[ { "path": ".editorconfig", "content": "# EditorConfig is awesome: https://editorconfig.org/\n\nroot = true\n\n[*]\nend_of_line = lf\ncharset = utf-8\nindent_style = space\nindent_size = 4\ninsert_final_newline = true\ntrim_trailing_whitespace = true\n" }, { "path": ".git-blame-ignore-revs", "content": "# Migrate code style to Black\n7957af562d5ce8266b177039783be4dc8bdd7898\n" }, { "path": ".gitattributes", "content": "borg/_version.py export-subst\n\n*.py diff=python\ndocs/usage/*.rst.inc merge=ours\ndocs/man/* merge=ours\n" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: borgbackup\nliberapay: borgbackup\nopen_collective: borgbackup\ncustom: ['https://www.borgbackup.org/support/fund.html']\n" }, { "path": ".github/ISSUE_TEMPLATE.md", "content": "\n\n## Have you checked the BorgBackup docs, FAQ, and open GitHub issues?\n\nNo\n\n## Is this a bug/issue report or a question?\n\nBug/Issue/Question\n\n## System information. For client/server mode, post info for both machines.\n\n#### Your Borg version (borg -V).\n\n#### Operating system (distribution) and version.\n\n#### Hardware/network configuration and filesystems used.\n\n#### How much data is handled by Borg?\n\n#### Full Borg command line that led to the problem (leave out excludes and passwords).\n\n\n## Describe the problem you're observing.\n\n#### Can you reproduce the problem? If so, describe how. If not, describe troubleshooting steps you took before opening the issue.\n\n#### Include any warnings/errors/backtraces from the system logs\n\n\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "\n\n## Description\n\n\n\n\n## Checklist\n\n- [ ] PR is against `master` (or maintenance branch if only applicable there)\n- [ ] New code has tests and docs where appropriate\n- [ ] Tests pass (run `tox` or the relevant test subset)\n- [ ] Commit messages are clean and reference related issues\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n actions:\n patterns:\n - \"*\"\n - package-ecosystem: \"pip\"\n directory: \"/requirements.d\"\n ignore:\n - dependency-name: \"black\"\n schedule:\n interval: \"weekly\"\n cooldown:\n semver-major-days: 90\n semver-minor-days: 30\n groups:\n pip-dependencies:\n patterns:\n - \"*\"\n\n" }, { "path": ".github/workflows/backport.yml", "content": "name: Backport pull request\n\non:\n pull_request_target:\n types: [closed]\n issue_comment:\n types: [created]\n\npermissions:\n contents: write # so it can comment\n pull-requests: write # so it can create pull requests\n\njobs:\n backport:\n name: Backport pull request\n runs-on: ubuntu-24.04\n timeout-minutes: 5\n\n # Only run when pull request is merged\n # or when a comment starting with `/backport` is created by someone other than the\n # https://github.com/backport-action bot user (user id: 97796249). Note that if you use your\n # own PAT as `github_token`, that you should replace this id with yours.\n if: >\n (\n github.event_name == 'pull_request_target' &&\n github.event.pull_request.merged\n ) || (\n github.event_name == 'issue_comment' &&\n github.event.issue.pull_request &&\n github.event.comment.user.id != 97796249 &&\n startsWith(github.event.comment.body, '/backport')\n )\n steps:\n - uses: actions/checkout@v6\n - name: Create backport pull requests\n uses: korthout/backport-action@v4\n with:\n label_pattern: '^port/(.+)$'\n" }, { "path": ".github/workflows/black.yaml", "content": "# https://black.readthedocs.io/en/stable/integrations/github_actions.html#usage\n# See also what we use locally in requirements.d/codestyle.txt — this should be the same version here.\n\nname: Lint\n\non:\n push:\n paths:\n - '**.py'\n - 'pyproject.toml'\n - '.github/workflows/black.yaml'\n pull_request:\n paths:\n - '**.py'\n - 'pyproject.toml'\n - '.github/workflows/black.yaml'\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}\n cancel-in-progress: ${{ github.event_name == 'pull_request' }}\n\njobs:\n lint:\n runs-on: ubuntu-22.04\n timeout-minutes: 5\n steps:\n - uses: actions/checkout@v6\n - uses: psf/black@c6755bb741b6481d6b3d3bb563c83fa060db96c9 # 26.3.1\n with:\n version: \"~= 24.0\"\n" }, { "path": ".github/workflows/canary.yml", "content": "name: Canary (Unlocked Requirements)\n\non:\n schedule:\n - cron: '0 7 * * *' # Run at 07:00 UTC\n workflow_dispatch: # Allow manual trigger\n\npermissions:\n contents: read\n\njobs:\n canary_tests:\n name: Canary (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.toxenv }})\n runs-on: ${{ matrix.os }}\n timeout-minutes: 360\n strategy:\n fail-fast: false\n matrix:\n include:\n # A representative subset of environments\n - os: ubuntu-22.04\n python-version: '3.10'\n toxenv: py310-llfuse\n - os: ubuntu-24.04\n python-version: '3.12'\n toxenv: py312-pyfuse3\n - os: ubuntu-24.04\n python-version: '3.14'\n toxenv: py314-mfusepy\n - os: macos-15\n python-version: '3.14'\n toxenv: py314-none\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0\n fetch-tags: true\n\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n\n - name: Install Linux packages\n if: ${{ runner.os == 'Linux' }}\n shell: bash\n run: |\n sudo apt-get update\n sudo apt-get install -y pkg-config build-essential\n sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev\n if [[ \"${{ matrix.toxenv }}\" == *\"llfuse\"* ]]; then\n sudo apt-get install -y libfuse-dev fuse\n elif [[ \"${{ matrix.toxenv }}\" == *\"pyfuse3\"* || \"${{ matrix.toxenv }}\" == *\"mfusepy\"* ]]; then\n sudo apt-get install -y libfuse3-dev fuse3\n fi\n\n - name: Install macOS packages\n if: ${{ runner.os == 'macOS' }}\n shell: bash\n run: |\n brew bundle install || true\n\n - name: Install Python requirements (UNLOCKED)\n shell: bash\n run: |\n python -m pip install --upgrade pip setuptools wheel\n # Use UNLOCKED requirements to catch upstream breakages\n pip install -r requirements.d/development.txt\n\n - name: Install borgbackup\n shell: bash\n run: |\n if [[ \"${{ matrix.toxenv }}\" == *\"llfuse\"* ]]; then\n pip install -e \".[llfuse,cockpit]\"\n elif [[ \"${{ matrix.toxenv }}\" == *\"pyfuse3\"* ]]; then\n pip install -e \".[pyfuse3,cockpit]\"\n elif [[ \"${{ matrix.toxenv }}\" == *\"mfusepy\"* ]]; then\n pip install -e \".[mfusepy,cockpit]\"\n else\n pip install -e \".[cockpit]\"\n fi\n\n - name: Run tests (Canary)\n shell: bash\n run: |\n if [[ \"${{ matrix.toxenv }}\" == *\"-windows\" ]]; then\n python -m pytest -n4 --benchmark-skip -vv -rs -k \"not remote\" --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml\n else\n # Force tox to use the unlocked requirements in its environment creation\n # by overriding the deps if possible, or just trusting it uses development.txt\n # which we already installed in the root. Actually tox creates its own venv.\n # We need to tell tox to use the unlocked file.\n tox -e ${{ matrix.toxenv }} --override \"env_run_base.deps=[-rrequirements.d/development.txt]\"\n fi\n\n windows_canary:\n\n if: true # can be used to temporarily disable the build\n name: Canary (Windows)\n runs-on: windows-latest\n timeout-minutes: 180\n\n env:\n PY_COLORS: 1\n\n defaults:\n run:\n shell: msys2 {0}\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - uses: msys2/setup-msys2@v2\n with:\n msystem: UCRT64\n update: true\n\n - name: Install system packages\n run: ./scripts/msys2-install-deps development\n\n - name: Build python venv\n run: |\n # building cffi / argon2-cffi in the venv fails, so we try to use the system packages\n python -m venv --system-site-packages env\n . env/bin/activate\n # python -m pip install --upgrade pip\n # pip install --upgrade setuptools build wheel\n pip install -r requirements.d/pyinstaller.txt\n\n - name: Build\n run: |\n # build borg.exe\n . env/bin/activate\n pip install -e \".[cockpit,s3,sftp,rest,rclone]\"\n mkdir -p dist/binary\n pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec\n # build sdist and wheel in dist/...\n python -m build\n\n - name: Run tests\n run: |\n # Ensure locally built binary in ./dist/binary/borg-dir is found during tests\n export PATH=\"$GITHUB_WORKSPACE/dist/binary/borg-dir:$PATH\"\n borg.exe -V\n . env/bin/activate\n python -m pytest -n4 --benchmark-skip -vv -rs -k \"not remote\" --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml\n" }, { "path": ".github/workflows/ci.yml", "content": "# badge: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=master\n\nname: CI\n\non:\n push:\n branches: [ master ]\n tags:\n - '2.*'\n pull_request:\n branches: [ master ]\n paths:\n - '**.py'\n - '**.pyx'\n - '**.c'\n - '**.h'\n - '**.yml'\n - '**.toml'\n - '**.cfg'\n - '**.ini'\n - 'requirements.d/*'\n - '!docs/**'\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}\n cancel-in-progress: ${{ github.event_name == 'pull_request' }}\n\npermissions:\n contents: read\n\njobs:\n lint:\n\n runs-on: ubuntu-22.04\n timeout-minutes: 5\n\n steps:\n - uses: actions/checkout@v6\n - uses: astral-sh/ruff-action@v3\n\n security:\n\n runs-on: ubuntu-24.04\n timeout-minutes: 5\n\n steps:\n - uses: actions/checkout@v6\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.10'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install bandit[toml]\n - name: Run Bandit\n run: |\n bandit -r src/borg -c pyproject.toml\n\n asan_ubsan:\n\n runs-on: ubuntu-24.04\n timeout-minutes: 25\n needs: [lint]\n\n steps:\n - uses: actions/checkout@v6\n with:\n # Just fetching one commit is not enough for setuptools-scm, so we fetch all.\n fetch-depth: 0\n fetch-tags: true\n\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.12'\n\n - name: Install system packages\n run: |\n sudo apt-get update\n sudo apt-get install -y pkg-config build-essential\n sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev\n\n - name: Install Python dependencies\n run: |\n python -m pip install --upgrade pip\n pip install -r requirements.d/development.lock.txt\n\n - name: Build Borg with ASan/UBSan\n # Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.\n # How this works:\n # - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit\n # (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints\n # a detailed error report to stderr, including a stack trace, and forces the process to exit with\n # non-zero status. In CI, this will fail the step/job so you will notice.\n # - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings).\n env:\n CFLAGS: \"-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined\"\n CXXFLAGS: \"-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined\"\n LDFLAGS: \"-fsanitize=address,undefined\"\n # ASAN_OPTIONS controls AddressSanitizer runtime tweaks:\n # - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests.\n # - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected.\n # - check_initialization_order=1: Catch uses that depend on static initialization order (C++).\n # - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead).\n ASAN_OPTIONS: \"detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1\"\n # UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime:\n # - print_stacktrace=1: Include a stack trace for UB reports to ease debugging.\n # Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to\n # abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS=\"print_stacktrace=1:halt_on_error=1\").\n UBSAN_OPTIONS: \"print_stacktrace=1\"\n # PYTHONDEVMODE enables additional Python runtime checks and warnings.\n PYTHONDEVMODE: \"1\"\n run: pip install -e .\n\n - name: Run tests under sanitizers\n env:\n ASAN_OPTIONS: \"detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1\"\n UBSAN_OPTIONS: \"print_stacktrace=1\"\n PYTHONDEVMODE: \"1\"\n # Ensure the ASan runtime is loaded first to avoid \"ASan runtime does not come first\" warnings.\n # We discover libasan/libubsan paths via gcc and preload them for the Python test process.\n # the remote tests are slow and likely won't find anything useful\n run: |\n set -euo pipefail\n export LD_PRELOAD=\"$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)\"\n echo \"Using LD_PRELOAD=$LD_PRELOAD\"\n pytest -v --benchmark-skip -k \"not remote\"\n\n native_tests:\n\n needs: [lint]\n permissions:\n contents: read\n id-token: write\n attestations: write\n strategy:\n fail-fast: true\n # noinspection YAMLSchemaValidation\n matrix: >-\n ${{ fromJSON(\n github.event_name == 'pull_request' && '{\n \"include\": [\n {\"os\": \"ubuntu-22.04\", \"python-version\": \"3.10\", \"toxenv\": \"mypy\"},\n {\"os\": \"ubuntu-22.04\", \"python-version\": \"3.11\", \"toxenv\": \"docs\"},\n {\"os\": \"ubuntu-22.04\", \"python-version\": \"3.10\", \"toxenv\": \"py310-llfuse\"},\n {\"os\": \"ubuntu-24.04\", \"python-version\": \"3.12\", \"toxenv\": \"py312-pyfuse3\"},\n {\"os\": \"ubuntu-24.04\", \"python-version\": \"3.14\", \"toxenv\": \"py314-mfusepy\"}\n ]\n }' || '{\n \"include\": [\n {\"os\": \"ubuntu-22.04\", \"python-version\": \"3.11\", \"toxenv\": \"py311-pyfuse3\", \"binary\": \"borg-linux-glibc235-x86_64-gh\"},\n {\"os\": \"ubuntu-22.04-arm\", \"python-version\": \"3.11\", \"toxenv\": \"py311-pyfuse3\", \"binary\": \"borg-linux-glibc235-arm64-gh\"},\n {\"os\": \"ubuntu-24.04\", \"python-version\": \"3.12\", \"toxenv\": \"py312-llfuse\"},\n {\"os\": \"ubuntu-24.04\", \"python-version\": \"3.13\", \"toxenv\": \"py313-pyfuse3\"},\n {\"os\": \"ubuntu-24.04\", \"python-version\": \"3.14\", \"toxenv\": \"py314-mfusepy\"},\n {\"os\": \"macos-15\", \"python-version\": \"3.11\", \"toxenv\": \"py311-none\", \"binary\": \"borg-macos-15-arm64-gh\"},\n {\"os\": \"macos-15-intel\", \"python-version\": \"3.11\", \"toxenv\": \"py311-none\", \"binary\": \"borg-macos-15-x86_64-gh\"}\n ]\n }'\n ) }}\n env:\n TOXENV: ${{ matrix.toxenv }}\n\n runs-on: ${{ matrix.os }}\n # macOS machines can be slow, if overloaded.\n timeout-minutes: 360\n\n steps:\n - uses: actions/checkout@v6\n with:\n # Just fetching one commit is not enough for setuptools-scm, so we fetch all.\n fetch-depth: 0\n fetch-tags: true\n\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n\n - name: Cache pip\n uses: actions/cache@v5\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }}\n restore-keys: |\n ${{ runner.os }}-${{ runner.arch }}-pip-\n ${{ runner.os }}-${{ runner.arch }}-\n\n - name: Cache tox environments\n uses: actions/cache@v5\n with:\n path: .tox\n key: ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.lock.txt', 'pyproject.toml') }}\n restore-keys: |\n ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-\n ${{ runner.os }}-${{ runner.arch }}-tox-\n\n - name: Install Linux packages\n if: ${{ runner.os == 'Linux' }}\n run: |\n sudo apt-get update\n sudo apt-get install -y pkg-config build-essential\n sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev\n sudo apt-get install -y bash zsh fish # for shell completion tests\n sudo apt-get install -y rclone openssh-server curl\n if [[ \"$TOXENV\" == *\"llfuse\"* ]]; then\n sudo apt-get install -y libfuse-dev fuse # Required for Python llfuse module\n elif [[ \"$TOXENV\" == *\"pyfuse3\"* || \"$TOXENV\" == *\"mfusepy\"* ]]; then\n sudo apt-get install -y libfuse3-dev fuse3 # Required for Python pyfuse3 module\n fi\n\n - name: Install macOS packages\n if: ${{ runner.os == 'macOS' }}\n run: |\n brew unlink pkg-config@0.29.2 || true\n brew bundle install\n\n - name: Configure OpenSSH SFTP server (test only)\n if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}\n run: |\n sudo mkdir -p /run/sshd\n sudo useradd -m -s /bin/bash sftpuser || true\n # Create SSH key for the CI user and authorize it for sftpuser\n mkdir -p ~/.ssh\n chmod 700 ~/.ssh\n test -f ~/.ssh/id_ed25519 || ssh-keygen -t ed25519 -N '' -f ~/.ssh/id_ed25519\n sudo mkdir -p /home/sftpuser/.ssh\n sudo chmod 700 /home/sftpuser/.ssh\n sudo cp ~/.ssh/id_ed25519.pub /home/sftpuser/.ssh/authorized_keys\n sudo chown -R sftpuser:sftpuser /home/sftpuser/.ssh\n sudo chmod 600 /home/sftpuser/.ssh/authorized_keys\n # Allow publickey auth and enable Subsystem sftp\n sudo sed -i 's/^#\\?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config\n sudo sed -i 's/^#\\?PubkeyAuthentication .*/PubkeyAuthentication yes/' /etc/ssh/sshd_config\n if ! grep -q '^Subsystem sftp' /etc/ssh/sshd_config; then echo 'Subsystem sftp /usr/lib/openssh/sftp-server' | sudo tee -a /etc/ssh/sshd_config; fi\n # Ensure host keys exist to avoid slow generation on first sshd start\n sudo ssh-keygen -A\n # Start sshd (listen on default 22 inside runner)\n sudo /usr/sbin/sshd -D &\n # Add host key to known_hosts so paramiko trusts it\n ssh-keyscan -H localhost 127.0.0.1 | tee -a ~/.ssh/known_hosts\n # Start ssh-agent and add our key so paramiko can use the agent\n eval \"$(ssh-agent -s)\"\n ssh-add ~/.ssh/id_ed25519\n # Export SFTP test URL for tox via GITHUB_ENV\n echo \"BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo\" >> $GITHUB_ENV\n\n - name: Install and configure MinIO S3 server (test only)\n if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}\n run: |\n set -e\n arch=$(uname -m)\n case \"$arch\" in\n x86_64|amd64) srv_url=https://dl.min.io/server/minio/release/linux-amd64/minio; cli_url=https://dl.min.io/client/mc/release/linux-amd64/mc ;;\n aarch64|arm64) srv_url=https://dl.min.io/server/minio/release/linux-arm64/minio; cli_url=https://dl.min.io/client/mc/release/linux-arm64/mc ;;\n *) echo \"Unsupported arch: $arch\"; exit 1 ;;\n esac\n curl -fsSL -o /usr/local/bin/minio \"$srv_url\"\n curl -fsSL -o /usr/local/bin/mc \"$cli_url\"\n sudo chmod +x /usr/local/bin/minio /usr/local/bin/mc\n export PATH=/usr/local/bin:$PATH\n # Start MinIO on :9000 with default credentials (minioadmin/minioadmin)\n MINIO_DIR=\"$GITHUB_WORKSPACE/.minio-data\"\n MINIO_LOG=\"$GITHUB_WORKSPACE/.minio.log\"\n mkdir -p \"$MINIO_DIR\"\n nohup minio server \"$MINIO_DIR\" --address \":9000\" >\"$MINIO_LOG\" 2>&1 &\n # Wait for MinIO port to be ready\n for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done\n # Configure client and create bucket\n mc alias set local http://127.0.0.1:9000 minioadmin minioadmin\n mc mb --ignore-existing local/borg\n # Export S3 test URL for tox via GITHUB_ENV\n echo \"BORG_TEST_S3_REPO=s3:minioadmin:minioadmin@http://127.0.0.1:9000/borg/s3-repo\" >> $GITHUB_ENV\n\n - name: Install Python requirements\n run: |\n python -m pip install --upgrade pip setuptools wheel\n pip install -r requirements.d/development.lock.txt\n\n - name: Install borgbackup\n run: |\n if [[ \"$TOXENV\" == *\"llfuse\"* ]]; then\n pip install -ve \".[llfuse,cockpit,s3,sftp,rest,rclone]\"\n elif [[ \"$TOXENV\" == *\"pyfuse3\"* ]]; then\n pip install -ve \".[pyfuse3,cockpit,s3,sftp,rest,rclone]\"\n elif [[ \"$TOXENV\" == *\"mfusepy\"* ]]; then\n pip install -ve \".[mfusepy,cockpit,s3,sftp,rest,rclone]\"\n else\n pip install -ve \".[cockpit,s3,sftp,rest,rclone]\"\n fi\n\n - name: Build Borg fat binaries (${{ matrix.binary }})\n if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}\n run: |\n pip install -r requirements.d/pyinstaller.txt\n mkdir -p dist/binary\n pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec\n\n - name: Smoke-test the built binary (${{ matrix.binary }})\n if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}\n run: |\n pushd dist/binary\n echo \"single-file binary\"\n chmod +x borg.exe\n ./borg.exe -V\n echo \"single-directory binary\"\n chmod +x borg-dir/borg.exe\n ./borg-dir/borg.exe -V\n tar czf borg.tgz borg-dir\n popd\n # Ensure locally built binary in ./dist/binary/borg-dir is found during tests\n export PATH=\"$GITHUB_WORKSPACE/dist/binary/borg-dir:$PATH\"\n echo \"borg.exe binary in PATH\"\n borg.exe -V\n\n - name: Prepare binaries (${{ matrix.binary }})\n if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}\n run: |\n mkdir -p artifacts\n if [ -f dist/binary/borg.exe ]; then\n cp dist/binary/borg.exe artifacts/${{ matrix.binary }}\n fi\n if [ -f dist/binary/borg.tgz ]; then\n cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz\n fi\n echo \"binary files\"\n ls -l artifacts/\n\n - name: Attest binaries provenance (${{ matrix.binary }})\n if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}\n uses: actions/attest-build-provenance@v4\n with:\n subject-path: 'artifacts/*'\n\n - name: Upload binaries (${{ matrix.binary }})\n if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}\n uses: actions/upload-artifact@v7\n with:\n name: ${{ matrix.binary }}\n path: artifacts/*\n if-no-files-found: error\n\n - name: run tox env\n run: |\n # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482.\n #sudo -E bash -c \"tox -e py\"\n # Ensure locally built binary in ./dist/binary/borg-dir is found during tests\n export PATH=\"$GITHUB_WORKSPACE/dist/binary/borg-dir:$PATH\"\n tox --skip-missing-interpreters\n\n - name: Upload test results to Codecov\n if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ runner.os }}\n python: ${{ matrix.python-version }}\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: test_results\n env_vars: OS,python\n files: test-results.xml\n\n - name: Upload coverage to Codecov\n if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ runner.os }}\n python: ${{ matrix.python-version }}\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: coverage\n env_vars: OS,python\n files: coverage.xml\n\n vm_tests:\n permissions:\n contents: read\n id-token: write\n attestations: write\n runs-on: ubuntu-24.04\n timeout-minutes: 180\n needs: [lint]\n continue-on-error: true\n\n strategy:\n fail-fast: false\n matrix:\n include:\n - os: freebsd\n version: '14.3'\n display_name: FreeBSD\n # Controls binary build and provenance attestation on tags\n do_binaries: true\n artifact_prefix: borg-freebsd-14-x86_64-gh\n\n - os: netbsd\n version: '10.1'\n display_name: NetBSD\n do_binaries: false\n\n - os: openbsd\n version: '7.8'\n display_name: OpenBSD\n do_binaries: false\n\n - os: omnios\n version: 'r151056'\n display_name: OmniOS\n do_binaries: false\n\n - os: haiku\n version: 'r1beta5'\n display_name: Haiku\n do_binaries: false\n\n steps:\n - name: Check out repository\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n fetch-tags: true\n\n - name: Test on ${{ matrix.display_name }}\n id: cross_os\n uses: cross-platform-actions/action@v0.32.0\n env:\n DO_BINARIES: ${{ matrix.do_binaries }}\n with:\n operating_system: ${{ matrix.os }}\n version: ${{ matrix.version }}\n shell: bash\n run: |\n set -euxo pipefail\n\n case \"${{ matrix.os }}\" in\n freebsd)\n export IGNORE_OSVERSION=yes\n sudo -E pkg update -f\n sudo -E pkg install -y liblz4 pkgconf\n sudo -E pkg install -y fusefs-libs\n sudo -E kldload fusefs\n sudo -E sysctl vfs.usermount=1\n sudo -E chmod 666 /dev/fuse\n sudo -E pkg install -y rust\n sudo -E pkg install -y gmake\n sudo -E pkg install -y git\n sudo -E pkg install -y python310 py310-sqlite3\n sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv\n sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3\n sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python\n sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3\n sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip\n\n # required for libsodium/pynacl build\n export MAKE=gmake\n\n python -m venv .venv\n . .venv/bin/activate\n python -V\n pip -V\n python -m pip install --upgrade pip wheel\n pip install -r requirements.d/development.lock.txt\n pip install -e \".[mfusepy,cockpit,s3,sftp,rest,rclone]\"\n tox -e py311-mfusepy\n\n if [[ \"${{ matrix.do_binaries }}\" == \"true\" && \"${{ startsWith(github.ref, 'refs/tags/') }}\" == \"true\" ]]; then\n python -m pip install -r requirements.d/pyinstaller.txt\n mkdir -p dist/binary\n pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec\n pushd dist/binary\n echo \"single-file binary\"\n chmod +x borg.exe\n ./borg.exe -V\n echo \"single-directory binary\"\n chmod +x borg-dir/borg.exe\n ./borg-dir/borg.exe -V\n tar czf borg.tgz borg-dir\n popd\n mkdir -p artifacts\n if [ -f dist/binary/borg.exe ]; then\n cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }}\n fi\n if [ -f dist/binary/borg.tgz ]; then\n cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz\n fi\n fi\n ;;\n\n netbsd)\n arch=\"$(uname -m)\"\n sudo -E mkdir -p /usr/pkg/etc/pkgin\n echo \"https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All\" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null\n sudo -E pkgin update\n sudo -E pkgin -y upgrade\n sudo -E pkgin -y install lz4 git\n sudo -E pkgin -y install rust\n sudo -E pkgin -y install pkg-config\n sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox\n sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3\n sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3\n sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3\n sudo -E ln -sf /usr/pkg/bin/tox-3.11 /usr/pkg/bin/tox3\n\n # Ensure base system admin tools are on PATH for the non-root shell\n export PATH=\"/sbin:/usr/sbin:$PATH\"\n\n echo \"--- Preparing an extattr-enabled filesystem ---\"\n # On many NetBSD setups /tmp is tmpfs without extended attributes.\n # Create a FFS image with extended attributes enabled and use it for TMPDIR.\n VNDDEV=\"vnd0\"\n IMGFILE=\"/tmp/fs.img\"\n sudo -E dd if=/dev/zero of=${IMGFILE} bs=1m count=1024\n sudo -E vndconfig -c \"${VNDDEV}\" \"${IMGFILE}\"\n sudo -E newfs -O 2ea /dev/r${VNDDEV}a\n MNT=\"/mnt/eafs\"\n sudo -E mkdir -p ${MNT}\n sudo -E mount -t ffs -o extattr /dev/${VNDDEV}a $MNT\n export TMPDIR=\"${MNT}/tmp\"\n sudo -E mkdir -p ${TMPDIR}\n sudo -E chmod 1777 ${TMPDIR}\n\n touch ${TMPDIR}/testfile\n lsextattr user ${TMPDIR}/testfile && echo \"[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***\"\n\n tox3 -e py311-none\n ;;\n\n openbsd)\n sudo -E pkg_add lz4 git\n sudo -E pkg_add rust\n sudo -E pkg_add openssl%3.5\n sudo -E pkg_add py3-pip py3-virtualenv py3-tox\n\n export BORG_OPENSSL_NAME=eopenssl35\n tox -e py312-none\n ;;\n\n omnios)\n sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils\n sudo ln -sf /usr/bin/python3.13 /usr/bin/python3\n sudo ln -sf /usr/bin/python3.13-config /usr/bin/python3-config\n sudo python3 -m ensurepip\n sudo python3 -m pip install virtualenv\n\n python3 -m venv .venv\n . .venv/bin/activate\n python -V\n pip -V\n python -m pip install --upgrade pip wheel\n pip install -r requirements.d/development.lock.txt\n # no fuse support on omnios in our tests usually\n pip install -e .\n\n tox -e py313-none\n ;;\n\n haiku)\n pkgman refresh\n pkgman install -y git pkgconfig lz4\n pkgman install -y openssl3\n pkgman install -y rust_bin\n pkgman install -y python3.10\n pkgman install -y cffi\n pkgman install -y lz4_devel openssl3_devel libffi_devel\n\n # there is no pkgman package for tox, so we install it into a venv\n python3 -m ensurepip --upgrade\n python3 -m pip install --upgrade pip wheel\n python3 -m venv .venv\n . .venv/bin/activate\n\n export PKG_CONFIG_PATH=\"/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}\"\n export BORG_LIBLZ4_PREFIX=/system/develop\n export BORG_OPENSSL_PREFIX=/system/develop\n pip install -r requirements.d/development.lock.txt\n pip install -e .\n\n # troubles with either tox or pytest xdist, so we run pytest manually:\n pytest -v -n auto -rs --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml --benchmark-skip -k \"not remote and not socket\"\n ;;\n esac\n\n - name: Upload artifacts\n if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries\n uses: actions/upload-artifact@v7\n with:\n name: ${{ matrix.artifact_prefix }}\n path: artifacts/*\n if-no-files-found: ignore\n\n - name: Attest provenance\n if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries\n uses: actions/attest-build-provenance@v4\n with:\n subject-path: 'artifacts/*'\n\n - name: Upload test results to Codecov\n if: ${{ !cancelled() }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ matrix.os }}\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: test_results\n env_vars: OS\n files: test-results.xml\n\n - name: Upload coverage to Codecov\n if: ${{ !cancelled() }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ matrix.os }}\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: coverage\n env_vars: OS\n files: coverage.xml\n\n windows_tests:\n\n if: true # can be used to temporarily disable the build\n runs-on: windows-latest\n timeout-minutes: 90\n needs: [lint]\n\n env:\n PY_COLORS: 1\n MSYS2_ARG_CONV_EXCL: \"*\"\n MSYS2_ENV_CONV_EXCL: \"*\"\n\n defaults:\n run:\n shell: msys2 {0}\n\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - uses: msys2/setup-msys2@v2\n with:\n msystem: UCRT64\n update: true\n\n - name: Install system packages\n run: ./scripts/msys2-install-deps development\n\n - name: Build python venv\n run: |\n # building cffi / argon2-cffi in the venv fails, so we try to use the system packages\n python -m venv --system-site-packages env\n . env/bin/activate\n # python -m pip install --upgrade pip\n # pip install --upgrade setuptools build wheel\n pip install -r requirements.d/pyinstaller.txt\n\n - name: Build\n run: |\n # build borg.exe\n . env/bin/activate\n pip install -e \".[cockpit,s3,sftp,rest,rclone]\"\n mkdir -p dist/binary\n pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec\n # build sdist and wheel in dist/...\n python -m build\n\n - uses: actions/upload-artifact@v7\n with:\n name: borg-windows\n path: dist/binary/borg.exe\n\n - name: Run tests\n run: |\n # Ensure locally built binary in ./dist/binary/borg-dir is found during tests\n export PATH=\"$GITHUB_WORKSPACE/dist/binary/borg-dir:$PATH\"\n borg.exe -V\n . env/bin/activate\n python -m pytest -n4 --benchmark-skip -vv -rs -k \"not remote\" --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml\n\n - name: Upload test results to Codecov\n if: ${{ !cancelled() }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ runner.os }}\n python: '3.11'\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: test_results\n env_vars: OS,python\n files: test-results.xml\n\n - name: Upload coverage to Codecov\n if: ${{ !cancelled() }}\n uses: codecov/codecov-action@v5\n env:\n OS: ${{ runner.os }}\n python: '3.11'\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n report_type: coverage\n env_vars: OS,python\n files: coverage.xml\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "# CodeQL semantic code analysis engine\n\nname: \"CodeQL\"\n\non:\n push:\n branches: [ master ]\n paths:\n - '**.py'\n - '**.pyx'\n - '**.c'\n - '**.h'\n - '.github/workflows/codeql-analysis.yml'\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [ master ]\n paths:\n - '**.py'\n - '**.pyx'\n - '**.c'\n - '**.h'\n - '.github/workflows/codeql-analysis.yml'\n schedule:\n - cron: '39 2 * * 5'\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}\n cancel-in-progress: ${{ github.event_name == 'pull_request' }}\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-24.04\n timeout-minutes: 20\n permissions:\n actions: read\n contents: read\n security-events: write\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'cpp', 'python' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]\n # Learn more about CodeQL language support at https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n with:\n # Just fetching one commit is not enough for setuptools-scm, so we fetch all.\n fetch-depth: 0\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: 3.11\n - name: Cache pip\n uses: actions/cache@v5\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}\n restore-keys: |\n ${{ runner.os }}-pip-\n ${{ runner.os }}-\n - name: Install requirements\n run: |\n sudo apt-get update\n sudo apt-get install -y pkg-config build-essential\n sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v4\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n - name: Build and install Borg\n run: |\n python3 -m venv ../borg-env\n source ../borg-env/bin/activate\n pip3 install -r requirements.d/development.txt\n pip3 install -ve .\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v4\n" }, { "path": ".gitignore", "content": "MANIFEST\ndocs/_build\nbuild\ndist\n.tox\nsrc/borg/compress.c\nsrc/borg/hashindex.c\nsrc/borg/crypto/low_level.c\nsrc/borg/item.c\nsrc/borg/chunkers/buzhash.c\nsrc/borg/chunkers/buzhash64.c\nsrc/borg/chunkers/reader.c\nsrc/borg/checksums.c\nsrc/borg/platform/darwin.c\nsrc/borg/platform/freebsd.c\nsrc/borg/platform/netbsd.c\nsrc/borg/platform/linux.c\nsrc/borg/platform/syncfilerange.c\nsrc/borg/platform/posix.c\nsrc/borg/platform/windows.c\nsrc/borg/_version.py\n*.egg-info\n*.pyc\n*.pyd\n*.so\n.idea/\n.junie/\n.vscode/\nborg.exe\n.coverage\n.coverage.*\ncoverage.xml\ntest-results.xml\n.vagrant\n.DS_Store\n" }, { "path": ".mailmap", "content": "Abdel-Rahman \nBrian Johnson \nCarlo Teubner \nMark Edgington \nLeo Famulari \nMarian Beermann \nThomas Waldmann \nDan Christensen \nAntoine Beaupré \nHartmut Goebel \nMichael Gajda \nMilkey Mouse \nRonny Pfannschmidt \nStefan Tatschner \n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n- repo: https://github.com/psf/black\n rev: 24.8.0\n hooks:\n - id: black\n- repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.15.0\n hooks:\n - id: ruff\n" }, { "path": ".readthedocs.yaml", "content": "# .readthedocs.yaml - Read the Docs configuration file.\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details.\n\nversion: 2\n\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.11\"\n jobs:\n post_checkout:\n - git fetch --unshallow\n apt_packages:\n - build-essential\n - pkg-config\n - libacl1-dev\n - libssl-dev\n - liblz4-dev\n\npython:\n install:\n - requirements: requirements.d/development.lock.txt\n - requirements: requirements.d/docs.txt\n - method: pip\n path: .\n\nsphinx:\n configuration: docs/conf.py\n\nformats:\n - htmlzip\n" }, { "path": "AUTHORS", "content": "Email addresses listed here are not intended for support.\nPlease see the `support section`_ instead.\n\n.. _support section: https://borgbackup.readthedocs.io/en/stable/support.html\n\nBorg authors (\"The Borg Collective\")\n------------------------------------\n\n- Thomas Waldmann \n- Radek Podgorny \n- Yuri D'Elia\n- Michael Hanselmann \n- Teemu Toivanen \n- Marian Beermann \n- Martin Hostettler \n- Daniel Reichelt \n- Lauri Niskanen \n- Abdel-Rahman A. (Abogical)\n- Gu1nness \n- Andrey Andreyevich Bienkowski \n\nRetired\n```````\n\n- Antoine Beaupré \n\nBorg is a fork of Attic.\n\nAttic authors\n-------------\n\nAttic is written and maintained by Jonas Borgström and various contributors:\n\nAttic Development Lead\n``````````````````````\n- Jonas Borgström \n\nAttic Patches and Suggestions\n`````````````````````````````\n- Brian Johnson\n- Cyril Roussillon\n- Dan Christensen\n- Jeremy Maitin-Shepard\n- Johann Klähn\n- Petros Moisiadis\n- Thomas Waldmann\n" }, { "path": "Brewfile", "content": "brew 'pkgconf'\nbrew 'lz4'\nbrew 'openssl@3'\n\n# osxfuse (aka macFUSE) is only required for \"borg mount\",\n# but won't work on GitHub Actions' workers.\n# It requires installing a kernel extension, so some users\n# may want it and some won't.\n\n#cask 'osxfuse'\n" }, { "path": "LICENSE", "content": "Copyright (C) 2015-2025 The Borg Collective (see AUTHORS file)\nCopyright (C) 2010-2014 Jonas Borgström \nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n 1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n 2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n 3. The name of the author may not be used to endorse or promote\n products derived from this software without specific prior\n written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "MANIFEST.in", "content": "# The files we need to include in the sdist are handled automatically by\n# setuptools_scm - it includes all git-committed files.\n# But we want to exclude some committed files/directories not needed in the sdist:\nexclude .editorconfig .gitattributes .gitignore .mailmap Vagrantfile\nprune .github\ninclude src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c\ninclude src/borg/platform/syncfilerange.c\ninclude src/borg/platform/windows.c\n" }, { "path": "README.rst", "content": "This is borg2!\n--------------\n\nPlease note that this is the README for borg2 / master branch.\n\nFor the stable version's docs, please see here:\n\nhttps://borgbackup.readthedocs.io/en/stable/\n\nBorg2 is currently in beta testing and might get major and/or\nbreaking changes between beta releases (and there is no beta to\nnext-beta upgrade code, so you will have to delete and re-create repos).\n\nThus, **DO NOT USE BORG2 FOR YOUR PRODUCTION BACKUPS!** Please help with\ntesting it, but set it up *additionally* to your production backups.\n\nTODO: the screencasts need a remake using borg2, see here:\n\nhttps://github.com/borgbackup/borg/issues/6303\n\n\nWhat is BorgBackup?\n-------------------\n\nBorgBackup (short: Borg) is a deduplicating backup program.\nOptionally, it supports compression and authenticated encryption.\n\nThe main goal of Borg is to provide an efficient and secure way to back up data.\nThe data deduplication technique used makes Borg suitable for daily backups\nsince only changes are stored.\nThe authenticated encryption technique makes it suitable for backups to targets not\nfully trusted.\n\nSee the `installation manual`_ or, if you have already\ndownloaded Borg, ``docs/installation.rst`` to get started with Borg.\nThere is also an `offline documentation`_ available, in multiple formats.\n\n.. _installation manual: https://borgbackup.readthedocs.io/en/master/installation.html\n.. _offline documentation: https://readthedocs.org/projects/borgbackup/downloads\n\nMain features\n~~~~~~~~~~~~~\n\n**Space efficient storage**\n Deduplication based on content-defined chunking is used to reduce the number\n of bytes stored: each file is split into a number of variable length chunks\n and only chunks that have never been seen before are added to the repository.\n\n A chunk is considered duplicate if its id_hash value is identical.\n A cryptographically strong hash or MAC function is used as id_hash, e.g.\n (hmac-)sha256.\n\n To deduplicate, all the chunks in the same repository are considered, no\n matter whether they come from different machines, from previous backups,\n from the same backup or even from the same single file.\n\n Compared to other deduplication approaches, this method does NOT depend on:\n\n * file/directory names staying the same: So you can move your stuff around\n without killing the deduplication, even between machines sharing a repo.\n\n * complete files or time stamps staying the same: If a big file changes a\n little, only a few new chunks need to be stored - this is great for VMs or\n raw disks.\n\n * The absolute position of a data chunk inside a file: Stuff may get shifted\n and will still be found by the deduplication algorithm.\n\n**Speed**\n * performance-critical code (chunking, compression, encryption) is\n implemented in C/Cython\n * local caching\n * quick detection of unmodified files\n\n**Data encryption**\n All data can be protected client-side using 256-bit authenticated encryption\n (AES-OCB or chacha20-poly1305), ensuring data confidentiality, integrity and\n authenticity.\n\n**Obfuscation**\n Optionally, Borg can actively obfuscate, e.g., the size of files/chunks to\n make fingerprinting attacks more difficult.\n\n**Compression**\n All data can be optionally compressed:\n\n * lz4 (super fast, low compression)\n * zstd (wide range from high speed and low compression to high compression\n and lower speed)\n * zlib (medium speed and compression)\n * lzma (low speed, high compression)\n\n**Off-site backups**\n Borg can store data on any remote host accessible over SSH. If Borg is\n installed on the remote host, significant performance gains can be achieved\n compared to using a network file system (sshfs, NFS, ...).\n\n**Backups mountable as file systems**\n Backup archives are mountable as user-space file systems for easy interactive\n backup examination and restores (e.g., by using a regular file manager).\n\n**Easy installation on multiple platforms**\n We offer single-file binaries that do not require installing anything -\n you can just run them on these platforms:\n\n * Linux\n * macOS\n * FreeBSD\n * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet)\n * Cygwin (experimental, no binaries yet)\n * Windows Subsystem for Linux (WSL) on Windows 10/11 (experimental)\n\n**Free and Open Source Software**\n * security and functionality can be audited independently\n * licensed under the BSD (3-clause) license, see `License`_ for the\n complete license\n\nEasy to use\n~~~~~~~~~~~\n\nFor ease of use, set the BORG_REPO environment variable::\n\n $ export BORG_REPO=/path/to/repo\n\nCreate a new backup repository (see ``borg repo-create --help`` for encryption options)::\n\n $ borg repo-create -e repokey-aes-ocb\n\nCreate a new backup archive::\n\n $ borg create Monday1 ~/Documents\n\nNow do another backup, just to show off the great deduplication::\n\n $ borg create -v --stats Monday2 ~/Documents\n Repository: /path/to/repo\n Archive name: Monday2\n Archive fingerprint: 7714aef97c1a24539cc3dc73f79b060f14af04e2541da33d54c7ee8e81a00089\n Time (start): Mon, 2022-10-03 19:57:35 +0200\n Time (end): Mon, 2022-10-03 19:57:35 +0200\n Duration: 0.01 seconds\n Number of files: 24\n Original size: 29.73 MB\n Deduplicated size: 520 B\n\n\nHelping, donations and bounties, becoming a Patron\n--------------------------------------------------\n\nYour help is always welcome!\n\nSpread the word, give feedback, help with documentation, testing or development.\n\nYou can also give monetary support to the project, see here for details:\n\nhttps://www.borgbackup.org/support/fund.html\n\nLinks\n-----\n\n* `Main website `_\n* `Releases `_,\n `PyPI packages `_ and\n `Changelog `_\n* `Offline documentation `_\n* `GitHub `_ and\n `Issue tracker `_.\n* `Web chat (IRC) `_ and\n `Mailing list `_\n* `License `_\n* `Security contact `_\n\nCompatibility notes\n-------------------\n\nEXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER\nCHANGES (like when going from 0.x.y to 1.0.0 or from 1.x.y to 2.0.0).\n\nNOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.\n\nTHIS IS SOFTWARE IN DEVELOPMENT, DECIDE FOR YOURSELF WHETHER IT FITS YOUR NEEDS.\n\nSecurity issues should be reported to the `Security contact`_ (or\nsee ``docs/support.rst`` in the source distribution).\n\n.. start-badges\n\n|doc| |build| |coverage| |bestpractices|\n\n.. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=master\n :alt: Documentation\n :target: https://borgbackup.readthedocs.io/en/master/\n\n.. |build| image:: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=master\n :alt: Build Status (master)\n :target: https://github.com/borgbackup/borg/actions\n\n.. |coverage| image:: https://codecov.io/github/borgbackup/borg/coverage.svg?branch=master\n :alt: Test Coverage\n :target: https://codecov.io/github/borgbackup/borg?branch=master\n\n.. |screencast_basic| image:: https://asciinema.org/a/133292.png\n :alt: BorgBackup Basic Usage\n :target: https://asciinema.org/a/133292?autoplay=1&speed=1\n :width: 100%\n\n.. _installation: https://asciinema.org/a/133291?autoplay=1&speed=1\n\n.. _advanced usage: https://asciinema.org/a/133293?autoplay=1&speed=1\n\n.. |bestpractices| image:: https://bestpractices.coreinfrastructure.org/projects/271/badge\n :alt: Best Practices Score\n :target: https://bestpractices.coreinfrastructure.org/projects/271\n\n.. end-badges\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\n## Supported Versions\n\nThese Borg releases are currently supported with security updates.\n\n| Version | Supported |\n|---------|--------------------|\n| 2.0.x | :x: (beta) |\n| 1.4.x | :white_check_mark: |\n| 1.2.x | :x: (no new releases, critical fixes may still be backported) |\n| 1.1.x | :x: |\n| < 1.1 | :x: |\n\n## Reporting a Vulnerability\n\nSee here:\n\nhttps://borgbackup.readthedocs.io/en/latest/support.html#security-contact\n" }, { "path": "Vagrantfile", "content": "# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\n# Automated creation of testing environments/binaries on miscellaneous platforms\n\n$cpus = Integer(ENV.fetch('VMCPUS', '8')) # create VMs with that many cpus\n$xdistn = Integer(ENV.fetch('XDISTN', '8')) # dispatch tests to that many pytest workers\n$wmem = $xdistn * 256 # give the VM additional memory for workers [MB]\n\ndef packages_debianoid(user)\n return <<-EOF\n export DEBIAN_FRONTEND=noninteractive\n # this is to avoid grub asking about which device it should install to:\n echo \"set grub-pc/install_devices /dev/sda\" | debconf-communicate\n apt-get -y -qq update\n apt-get -y -qq dist-upgrade\n # for building borgbackup and dependencies:\n apt install -y pkg-config\n apt install -y libssl-dev libacl1-dev liblz4-dev || true\n apt install -y libfuse-dev fuse || true\n apt install -y libfuse3-dev fuse3 || true\n apt install -y locales || true\n sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen\n usermod -a -G fuse #{user}\n chgrp fuse /dev/fuse\n chmod 666 /dev/fuse\n apt install -y fakeroot build-essential git curl\n apt install -y python3-dev python3-setuptools virtualenv\n # for building python:\n apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev\n EOF\nend\n\ndef packages_freebsd\n return <<-EOF\n # in case the VM has no hostname set\n hostname freebsd\n # install all the (security and other) updates, base system\n freebsd-update --not-running-from-cron fetch install\n # for building borgbackup and dependencies:\n pkg install -y liblz4 pkgconf\n pkg install -y fusefs-libs || true\n pkg install -y fusefs-libs3 || true\n pkg install -y rust\n pkg install -y git bash # fakeroot causes lots of troubles on freebsd\n pkg install -y python310 py310-sqlite3\n pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv\n # make sure there is a python3/pip3/virtualenv command\n ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3\n ln -sf /usr/local/bin/pip-3.11 /usr/local/bin/pip3\n ln -sf /usr/local/bin/virtualenv-3.11 /usr/local/bin/virtualenv\n # make bash default / work:\n chsh -s bash vagrant\n mount -t fdescfs fdesc /dev/fd\n echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab\n # make FUSE work\n echo 'fuse_load=\"YES\"' >> /boot/loader.conf\n echo 'vfs.usermount=1' >> /etc/sysctl.conf\n kldload fusefs\n sysctl vfs.usermount=1\n pw groupmod operator -M vagrant\n # /dev/fuse has group operator\n chmod 666 /dev/fuse\n # install all the (security and other) updates, packages\n pkg update\n yes | pkg upgrade\n echo 'export BORG_OPENSSL_PREFIX=/usr' >> ~vagrant/.bash_profile\n # (re)mount / with acls\n mount -o acls /\n EOF\nend\n\ndef packages_openbsd\n return <<-EOF\n hostname \"openbsd77.localdomain\"\n echo \"$(hostname)\" > /etc/myname\n echo \"127.0.0.1\tlocalhost\" > /etc/hosts\n echo \"::1 localhost\" >> /etc/hosts\n echo \"127.0.0.1\t$(hostname) $(hostname -s)\" >> /etc/hosts\n echo \"https://ftp.eu.openbsd.org/pub/OpenBSD\" > /etc/installurl\n ftp https://cdn.openbsd.org/pub/OpenBSD/$(uname -r)/$(uname -m)/comp$(uname -r | tr -d .).tgz\n tar -C / -xzphf comp$(uname -r | tr -d .).tgz\n rm comp$(uname -r | tr -d .).tgz\n pkg_add bash\n chsh -s bash vagrant\n pkg_add lz4\n pkg_add git # no fakeroot\n pkg_add rust\n pkg_add openssl%3.4\n pkg_add py3-pip\n pkg_add py3-virtualenv\n echo 'export BORG_OPENSSL_NAME=eopenssl30' >> ~vagrant/.bash_profile\n EOF\nend\n\ndef packages_netbsd\n return <<-EOF\n echo 'https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf\n pkgin update\n pkgin -y upgrade\n pkg_add lz4 git\n pkg_add rust\n pkg_add bash\n chsh -s bash vagrant\n echo \"export PROMPT_COMMAND=\" >> ~vagrant/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen\n echo \"export PROMPT_COMMAND=\" >> ~root/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen\n pkg_add pkg-config\n # pkg_add fuse # llfuse supports netbsd, but is still buggy.\n # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space\n pkg_add py311-sqlite3 py311-pip py311-virtualenv py311-expat\n ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python\n ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python3\n ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip\n ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3\n ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv\n ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3\n EOF\nend\n\ndef package_update_openindiana\n return <<-EOF\n echo \"nameserver 1.1.1.1\" > /etc/resolv.conf\n # needs separate provisioning step + reboot to become effective:\n pkg update\n EOF\nend\n\ndef packages_openindiana\n return <<-EOF\n pkg install gcc-13 git\n pkg install pkg-config\n pkg install python-313\n ln -sf /usr/bin/python3.13 /usr/bin/python3\n ln -sf /usr/bin/python3.13-config /usr/bin/python3-config\n python3 -m ensurepip\n ln -sf /usr/bin/pip3.13 /usr/bin/pip3\n pip3 install virtualenv\n # let borg's pkg-config find openssl:\n pfexec pkg set-mediator -V 3 openssl\n EOF\nend\n\ndef install_pyenv(boxname)\n return <<-EOF\n echo 'export PYTHON_CONFIGURE_OPTS=\"${PYTHON_CONFIGURE_OPTS} --enable-shared\"' >> ~/.bash_profile\n echo 'export PYENV_ROOT=\"$HOME/.pyenv\"' >> ~/.bash_profile\n echo 'export PATH=\"$PYENV_ROOT/bin:$PATH\"' >> ~/.bash_profile\n . ~/.bash_profile\n curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash\n echo 'eval \"$(pyenv init --path)\"' >> ~/.bash_profile\n echo 'export PYENV_ROOT=\"$HOME/.pyenv\"' >> ~/.bashrc\n echo 'export PATH=\"$PYENV_ROOT/bin:$PATH\"' >> ~/.bashrc\n echo 'eval \"$(pyenv init -)\"' >> ~/.bashrc\n echo 'eval \"$(pyenv virtualenv-init -)\"' >> ~/.bashrc\n EOF\nend\n\ndef install_pythons(boxname)\n return <<-EOF\n . ~/.bash_profile\n echo \"PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}\"\n pyenv install 3.13.8\n pyenv rehash\n EOF\nend\n\ndef build_sys_venv(boxname)\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg\n virtualenv --python=python3 borg-env\n EOF\nend\n\ndef build_pyenv_venv(boxname)\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg\n # use the latest 3.13 release\n pyenv global 3.13.8\n pyenv virtualenv 3.13.8 borg-env\n ln -s ~/.pyenv/versions/borg-env .\n EOF\nend\n\ndef install_borg(fuse)\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg\n . borg-env/bin/activate\n pip install -U wheel # upgrade wheel, might be too old\n cd borg\n pip install -r requirements.d/development.lock.txt\n python3 scripts/make.py clean\n # install borgstore WITH all options, so it pulls in the needed\n # requirements, so they will also get into the binaries built. #8574\n pip install borgstore[sftp,s3]\n pip install -e .[#{fuse}]\n EOF\nend\n\ndef install_pyinstaller()\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg\n . borg-env/bin/activate\n pip install -r requirements.d/pyinstaller.txt\n EOF\nend\n\ndef build_binary_with_pyinstaller(boxname)\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg\n . borg-env/bin/activate\n cd borg\n pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec\n echo 'export PATH=\"/vagrant/borg:$PATH\"' >> ~/.bash_profile\n cd .. && tar -czvf borg.tgz borg-dir\n EOF\nend\n\ndef run_tests(boxname, skip_env)\n return <<-EOF\n . ~/.bash_profile\n cd /vagrant/borg/borg\n . ../borg-env/bin/activate\n if which pyenv 2> /dev/null; then\n # for testing, use the earliest point releases of the supported python versions:\n pyenv global 3.13.8\n pyenv local 3.13.8\n fi\n # otherwise: just use the system python\n # some OSes can only run specific test envs, e.g. because they miss FUSE support:\n export TOX_SKIP_ENV='#{skip_env}'\n if which fakeroot 2> /dev/null; then\n echo \"Running tox WITH fakeroot -u\"\n fakeroot -u tox --skip-missing-interpreters\n else\n echo \"Running tox WITHOUT fakeroot -u\"\n tox --skip-missing-interpreters\n fi\n EOF\nend\n\ndef fs_init(user)\n return <<-EOF\n # clean up (wrong/outdated) stuff we likely got via rsync:\n rm -rf /vagrant/borg/borg/.tox 2> /dev/null\n rm -rf /vagrant/borg/borg/borgbackup.egg-info 2> /dev/null\n rm -rf /vagrant/borg/borg/__pycache__ 2> /dev/null\n find /vagrant/borg/borg/src -name '__pycache__' -exec rm -rf {} \\\\; 2> /dev/null\n chown -R #{user} /vagrant/borg\n touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile\n echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile\n echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile\n echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile\n EOF\nend\n\nVagrant.configure(2) do |config|\n # use rsync to copy content to the folder\n config.vm.synced_folder \".\", \"/vagrant/borg/borg\", :type => \"rsync\", :rsync__args => [\"--verbose\", \"--archive\", \"--delete\", \"--exclude\", \".python-version\"], :rsync__chown => false\n # do not let the VM access . on the host machine via the default shared folder!\n config.vm.synced_folder \".\", \"/vagrant\", disabled: true\n\n config.vm.provider :virtualbox do |v|\n #v.gui = true\n v.cpus = $cpus\n end\n\n config.vm.define \"noble\" do |b|\n b.vm.box = \"bento/ubuntu-24.04\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_sys_venv(\"noble\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"noble\", \".*none.*\")\n end\n\n config.vm.define \"jammy\" do |b|\n b.vm.box = \"ubuntu/jammy64\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_sys_venv(\"jammy\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"jammy\", \".*none.*\")\n end\n\n config.vm.define \"trixie\" do |b|\n b.vm.box = \"debian/testing64\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"trixie\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"trixie\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"trixie\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"trixie\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"trixie\", \".*none.*\")\n end\n\n config.vm.define \"bookworm32\" do |b|\n b.vm.box = \"generic-x32/debian12\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"bookworm32\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"bookworm32\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"bookworm32\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"bookworm32\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"bookworm32\", \".*none.*\")\n end\n\n config.vm.define \"bookworm\" do |b|\n b.vm.box = \"debian/bookworm64\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"bookworm\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"bookworm\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"bookworm\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"bookworm\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"bookworm\", \".*none.*\")\n end\n\n config.vm.define \"bullseye\" do |b|\n b.vm.box = \"debian/bullseye64\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages debianoid\", :type => :shell, :inline => packages_debianoid(\"vagrant\")\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"bullseye\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"bullseye\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"bullseye\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"bullseye\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"bullseye\", \".*none.*\")\n end\n\n config.vm.define \"freebsd13\" do |b|\n b.vm.box = \"generic/freebsd13\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.ssh.shell = \"sh\"\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages freebsd\", :type => :shell, :inline => packages_freebsd\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"freebsd13\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"freebsd13\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"freebsd13\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"freebsd13\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"freebsd13\", \".*(pyfuse3|none).*\")\n end\n\n config.vm.define \"freebsd14\" do |b|\n b.vm.box = \"generic/freebsd14\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.ssh.shell = \"sh\"\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages freebsd\", :type => :shell, :inline => packages_freebsd\n b.vm.provision \"install pyenv\", :type => :shell, :privileged => false, :inline => install_pyenv(\"freebsd14\")\n b.vm.provision \"install pythons\", :type => :shell, :privileged => false, :inline => install_pythons(\"freebsd14\")\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_pyenv_venv(\"freebsd14\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"llfuse\")\n b.vm.provision \"install pyinstaller\", :type => :shell, :privileged => false, :inline => install_pyinstaller()\n b.vm.provision \"build binary with pyinstaller\", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller(\"freebsd14\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"freebsd14\", \".*(pyfuse3|none).*\")\n end\n\n config.vm.define \"openbsd7\" do |b|\n b.vm.box = \"l3system/openbsd77-amd64\"\n b.vm.provider :virtualbox do |v|\n v.memory = 1024 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages openbsd\", :type => :shell, :inline => packages_openbsd\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_sys_venv(\"openbsd7\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"nofuse\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"openbsd7\", \".*fuse.*\")\n end\n\n config.vm.define \"netbsd9\" do |b|\n b.vm.box = \"generic/netbsd9\"\n b.vm.provider :virtualbox do |v|\n v.memory = 4096 + $wmem # need big /tmp tmpfs in RAM!\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"packages netbsd\", :type => :shell, :inline => packages_netbsd\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_sys_venv(\"netbsd9\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"nofuse\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"netbsd9\", \".*fuse.*\")\n end\n\n # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of\n # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again.\n config.vm.define \"openindiana\" do |b|\n b.vm.box = \"openindiana/hipster\"\n b.vm.provider :virtualbox do |v|\n v.memory = 2048 + $wmem\n end\n b.vm.provision \"fs init\", :type => :shell, :inline => fs_init(\"vagrant\")\n b.vm.provision \"package update openindiana\", :type => :shell, :inline => package_update_openindiana, :reboot => true\n b.vm.provision \"packages openindiana\", :type => :shell, :inline => packages_openindiana\n b.vm.provision \"build env\", :type => :shell, :privileged => false, :inline => build_sys_venv(\"openindiana\")\n b.vm.provision \"install borg\", :type => :shell, :privileged => false, :inline => install_borg(\"nofuse\")\n b.vm.provision \"run tests\", :type => :shell, :privileged => false, :inline => run_tests(\"openindiana\", \".*fuse.*\")\n end\nend\n" }, { "path": "docs/3rd_party/README", "content": "Here we store third-party documentation, licenses, etc.\n\nPlease note that all files inside the \"borg\" package directory (except those\nexcluded in setup.py) will be installed, so do not keep docs or licenses\nthere.\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and an HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\nclean:\n\t-rm -rf $(BUILDDIR)/*\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/borg.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/borg.qhc\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/borg\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/borg\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\tmake -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n" }, { "path": "docs/_static/Makefile", "content": "\nall: logo.pdf logo.png\n\nlogo.pdf: logo.svg\n\tinkscape logo.svg --export-pdf=logo.pdf\n\nlogo.png: logo.svg\n\tinkscape logo.svg --export-png=logo.png --export-dpi=72,72\n\nclean:\n\trm -f logo.pdf logo.png\n" }, { "path": "docs/_static/logo_font.txt", "content": "Black Ops One\nJames Grieshaber\nSIL Open Font License, 1.1\n\nhttps://www.google.com/fonts/specimen/Black+Ops+One\n" }, { "path": "docs/_templates/globaltoc.html", "content": "
\n
\n {# Restrict the sidebar ToC depth to two levels while generating command usage pages.\n This avoids superfluous entries for each \"Description\" and \"Examples\" heading. #}\n {% if pagename.startswith(\"usage/\") and pagename not in (\n \"usage/general\", \"usage/help\", \"usage/debug\", \"usage/notes\",\n ) %}\n {% set maxdepth = 2 %}\n {% else %}\n {% set maxdepth = 3 %}\n {% endif %}\n\n {% set toctree = toctree(maxdepth=maxdepth, collapse=True) %}\n {% if toctree %}\n {{ toctree }}\n {% else %}\n {{ toc }}\n {% endif %}\n
\n
\n" }, { "path": "docs/_templates/layout.html", "content": "{%- extends \"basic/layout.html\" %}\n\n{# Do this so that Bootstrap is included before the main CSS file. #}\n{%- block htmltitle %}\n {% set script_files = script_files + [\"_static/myscript.js\"] %}\n \n \n \n \n \n \n \n {{ super() }}\n{%- endblock %}\n\n{%- block extrahead %}\n {% if theme_touch_icon %}\n \n {% endif %}\n \n {{ super() }}\n{% endblock %}\n\n{# Displays the URL for the homepage if it's set, or the master_doc if it is not. #}\n{% macro homepage() -%}\n {%- if theme_homepage %}\n {%- if hasdoc(theme_homepage) %}\n {{ pathto(theme_homepage) }}\n {%- else %}\n {{ theme_homepage }}\n {%- endif %}\n {%- else %}\n {{ pathto(master_doc) }}\n {%- endif %}\n{%- endmacro %}\n\n{# Displays the URL for the tospage if it's set, or falls back to the homepage macro. #}\n{% macro tospage() -%}\n {%- if theme_tospage %}\n {%- if hasdoc(theme_tospage) %}\n {{ pathto(theme_tospage) }}\n {%- else %}\n {{ theme_tospage }}\n {%- endif %}\n {%- else %}\n {{ homepage() }}\n {%- endif %}\n{%- endmacro %}\n\n{# Displays the URL for the projectpage if it's set, or falls back to the homepage macro. #}\n{% macro projectlink() -%}\n {%- if theme_projectlink %}\n {%- if hasdoc(theme_projectlink) %}\n {{ pathto(theme_projectlink) }}\n {%- else %}\n {{ theme_projectlink }}\n {%- endif %}\n {%- else %}\n {{ homepage() }}\n {%- endif %}\n{%- endmacro %}\n\n{# Displays the next and previous links both before and after the content. #}\n{% macro render_relations() -%}\n {% if prev or next %}\n
\n {% if prev %}\n
\n {{ prev.title }}\n
\n {% endif %}\n {%- if next and next.title != '<no title>' %}\n
\n {{ next.title }}\n
\n {%- endif %}\n
\n
\n {% endif %}\n{%- endmacro %}\n\n{%- macro guzzle_sidebar() %}\n
\n
\n {%- if sidebars != None %}\n {#- New-style sidebar: explicitly include/exclude templates. #}\n {%- for sidebartemplate in sidebars %}\n {%- include sidebartemplate %}\n {%- endfor %}\n {% else %}\n {% include \"logo-text.html\" %}\n {% include \"globaltoc.html\" %}\n {% include \"searchbox.html\" %}\n {%- endif %}\n
\n
\n{%- endmacro %}\n\n{%- block content %}\n\n {%- if pagename == 'index' and theme_index_template %}\n {% include theme_index_template %}\n {%- else %}\n
\n\n
\n \n
\n\n {%- block sidebar1 %}{{ guzzle_sidebar() }}{% endblock %}\n\n {%- block document_wrapper %}\n {%- block document %}\n
\n {% block breadcrumbs %}\n
\n
    \n
  1. Docs
    2. \n {% for doc in parents %}\n
  2. {{ doc.title }}
    3. \n {% endfor %}\n
  3. {{ title }}
    4. \n
\n
\n {% endblock %}\n
\n {% block body %} {% endblock %}\n
\n {%- block bottom_rel_links %}\n {{ render_relations() }}\n {%- endblock %}\n
\n
\n {%- endblock %}\n {%- endblock %}\n\n {%- block comments -%}\n {% if theme_disqus_comments_shortname %}\n
\n {% include \"comments.html\" %}\n
\n {% endif %}\n {%- endblock %}\n
\n {%- endif %}\n {%- endblock %}\n\n{%- block footer %}\n\n\n{%- block footer_wrapper %}\n
\n © Copyright {{ copyright }}. Created using Sphinx.\n
\n{%- endblock %}\n{%- block ga %}\n {%- if theme_google_analytics_account %}\n \n {%- endif %}\n{%- endblock %}\n{%- endblock %}\n" }, { "path": "docs/_templates/logo-text.html", "content": "\n \n\n {{ theme_project_nav_name or shorttitle }}\n\n" }, { "path": "docs/_templates/versionselector.html", "content": "
\n \n \n
\n\n" }, { "path": "docs/authors.rst", "content": ".. include:: global.rst.inc\n\nAuthors\n=======\n\n.. include:: ../AUTHORS\n\nLicense\n=======\n\n.. _license:\n\n.. include:: ../LICENSE\n :literal:\n" }, { "path": "docs/binaries/00_README.txt", "content": "Binary BorgBackup builds\n========================\n\nGeneral notes\n-------------\n\nThe binaries are supposed to work on the specified platform without installing anything else.\n\nThere are some limitations, though:\n- for Linux, your system must have the same or newer glibc version as the one used for building\n- for macOS, you need to have the same or newer macOS version as the one used for building\n- for other OSes, there are likely similar limitations\n\nIf you don't find something working on your system, check the older borg releases.\n\n*.asc are GnuPG signatures - only provided for locally built binaries.\n*.exe (or no extension) is the single-file fat binary.\n*.tgz is the single-directory fat binary (extract it once with tar -xzf).\n\nUsing the single-directory build is faster and does not require as much space\nin the temporary directory as the self-extracting single-file build.\n\nmacOS: to avoid issues, download the file via the command line OR remove the\n \"quarantine\" attribute after downloading:\n $ xattr -dr com.apple.quarantine borg-macos1012.tgz\n\n\nDownload the correct files\n--------------------------\n\nBinaries built on GitHub servers\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nborg-linux-glibc235-x86_64-gh Linux AMD/Intel (built on Ubuntu 22.04 LTS with glibc 2.35)\nborg-linux-glibc235-arm64-gh Linux ARM (built on Ubuntu 22.04 LTS with glibc 2.35)\n\nborg-macos-15-arm64-gh macOS Apple Silicon (built on macOS 15 w/o FUSE support)\nborg-macos-15-x86_64-gh macOS Intel (built on macOS 15 w/o FUSE support)\n\nborg-freebsd-14-x86_64-gh FreeBSD AMD/Intel (built on FreeBSD 14)\n\nBinaries built locally\n~~~~~~~~~~~~~~~~~~~~~~\n\nborg-linux-glibc231-x86_64 Linux (built on Debian 11 \"Bullseye\" with glibc 2.31)\n\nNote: if you don't find a specific binary here, check release 1.4.1 or 1.2.9.\n\nVerifying your download\n-----------------------\n\nI provide GPG signatures for files which I have built locally on my machines.\n\nTo check the GPG signature, download both the file and the corresponding\nsignature (*.asc file) and then (on the shell) type, for example:\n\n gpg --recv-keys 9F88FB52FAF7B393\n gpg --verify borgbackup.tar.gz.asc borgbackup.tar.gz\n\nThe files are signed by:\n\nThomas Waldmann \nGPG key fingerprint: 6D5B EF9A DD20 7580 5747 B70F 9F88 FB52 FAF7 B393\n\nMy fingerprint is also in the footer of all my BorgBackup mailing list posts.\n\n\nProvenance attestations for GitHub-built binaries\n-------------------------------------------------\n\nFor binaries built on GitHub (files with a \"-gh\" suffix in the name), we publish\nan artifact provenance attestation that proves the binary was built by our\nGitHub Actions workflow from a specific commit or tag. You can verify this using\nthe GitHub CLI (gh). Install it from https://cli.github.com/ and make sure you\nuse a recent version that supports \"gh attestation\".\n\nPractical example (Linux, 2.0.0b20 tag):\n\n curl -LO https://github.com/borgbackup/borg/releases/download/2.0.0b20/borg-linux-glibc235-x86_64-gh\n gh attestation verify --repo borgbackup/borg --source-ref refs/tags/2.0.0b20 borg-linux-glibc235-x86_64-gh\n\nIf verification succeeds, gh prints a summary stating the subject (your file),\nthat it was attested by GitHub Actions, and the job/workflow reference.\n\n\nInstalling\n----------\n\nIt is suggested that you rename or symlink the binary to just \"borg\".\nIf you need \"borgfs\", just also symlink it to the same binary; it will\ndetect internally under which name it was invoked.\n\nOn UNIX-like platforms, /usr/local/bin/ or ~/bin/ is a nice place for it,\nbut you can invoke it from anywhere by providing the full path to it.\n\nMake sure the file is readable and executable (chmod +rx borg on UNIX-like\nplatforms).\n\n\nReporting issues\n----------------\n\nPlease first check the FAQ and whether a GitHub issue already exists.\n\nIf you find a NEW issue, please open a ticket on our issue tracker:\n\nhttps://github.com/borgbackup/borg/issues/\n\nThere, please give:\n- the version number (it is displayed if you invoke borg -V)\n- the sha256sum of the binary\n- a good description of what the issue is\n- a good description of how to reproduce your issue\n- a traceback with system info (if you have one)\n- your precise platform (CPU, 32/64-bit?), OS, distribution, release\n- your Python and (g)libc versions\n\n" }, { "path": "docs/book.rst", "content": ":orphan:\n\n.. include:: global.rst.inc\n\nBorg documentation\n==================\n\n.. When you add an element here, do not forget to add it to index.rst.\n.. Note: Some things are in appendices (see latex_appendices in conf.py).\n\n.. toctree::\n :maxdepth: 2\n\n introduction\n installation\n quickstart\n usage\n deployment\n faq\n support\n changes\n internals\n development\n authors\n" }, { "path": "docs/borg_theme/css/borg.css", "content": "@import url(\"theme.css\");\n\ndt code {\n font-weight: normal;\n}\n\n#internals .toctree-wrapper > ul {\n column-count: 3;\n -webkit-column-count: 3;\n}\n\n#internals .toctree-wrapper > ul > li {\n display: inline-block;\n font-weight: bold;\n}\n\n#internals .toctree-wrapper > ul > li > ul {\n font-weight: normal;\n}\n\n/* bootstrap has a .container class which clashes with docutils' container class. */\n.docutils.container {\n width: auto;\n margin: 0;\n padding: 0;\n}\n\n/* the default (38px) produces a jumpy baseline in Firefox on Linux. */\nh1 {\n font-size: 36px;\n}\n\n.text-logo {\n background-color: #000200;\n color: #00dd00;\n}\n\n.text-logo:hover,\n.text-logo:active,\n.text-logo:focus {\n color: #5afe57;\n}\n\n/* by default the top and bottom margins are unequal which looks a bit unbalanced. */\n.sidebar-block {\n padding: 0;\n margin: 14px 0 24px 0;\n}\n\n#borg-documentation h1 + p .external img {\n width: 100%;\n}\n\n.container.experimental,\n#debugging-facilities {\n /* don't change text dimensions */\n margin: 0 -30px; /* padding below + border width */\n padding: 0 10px; /* 10 px visual margin between edge of text and the border */\n /* fallback for browsers that don't have repeating-linear-gradient: thick, red lines */\n border-left: 20px solid red;\n border-right: 20px solid red;\n /* fancy red stripes */\n border-image: repeating-linear-gradient(\n -45deg,rgba(255,0,0,0.1) 0,rgba(255,0,0,0.75) 10px,rgba(0,0,0,0) 10px,rgba(0,0,0,0) 20px,rgba(255,0,0,0.75) 20px) 0 20 repeat;\n}\n\n.topic {\n margin: 0 1em;\n padding: 0 1em;\n /* #4e4a4a = background of the ToC sidebar */\n border-left: 2px solid #4e4a4a;;\n border-right: 2px solid #4e4a4a;;\n}\n\ntable.docutils:not(.footnote) td,\ntable.docutils:not(.footnote) th {\n padding: .2em;\n}\n\ntable.docutils:not(.footnote) {\n border-collapse: collapse;\n border: none;\n}\n\ntable.docutils:not(.footnote) td,\ntable.docutils:not(.footnote) th {\n border: 1px solid #ddd;\n}\n\ntable.docutils:not(.footnote) tr:first-child th,\ntable.docutils:not(.footnote) tr:first-child td {\n border-top: 0;\n}\n\ntable.docutils:not(.footnote) tr:last-child td {\n border-bottom: 0;\n}\n\ntable.docutils:not(.footnote) tr td:first-child,\ntable.docutils:not(.footnote) tr th:first-child {\n border-left: 0;\n}\n\ntable.docutils:not(.footnote) tr td:last-child,\ntable.docutils:not(.footnote) tr th:last-child,\ntable.docutils.borg-options-table tr td {\n border-right: 0;\n}\n\ntable.docutils.option-list tr td,\ntable.docutils.borg-options-table tr td {\n border-left: 0;\n border-right: 0;\n}\n\ntable.docutils.borg-options-table tr td:first-child:not([colspan=\"3\"]) {\n border-top: 0;\n border-bottom: 0;\n}\n\n.borg-options-table td[colspan=\"3\"] p {\n margin: 0;\n}\n\n.borg-options-table {\n width: 100%;\n}\n\nkbd, /* used in usage pages for options */\ncode,\n.rst-content tt.literal,\n.rst-content tt.literal,\n.rst-content code.literal,\n.rst-content tt,\n.rst-content code,\np .literal,\np .literal span {\n border: none;\n padding: 0;\n color: black; /* slight contrast with #404040 of regular text */\n background: none;\n}\n\nkbd {\n box-shadow: none;\n line-height: 23px;\n word-wrap: normal;\n font-size: 15px;\n font-family: Consolas, monospace;\n}\n\n.borg-options-table tr td:nth-child(2) .pre {\n white-space: nowrap;\n}\n\n.borg-options-table tr td:first-child {\n width: 2em;\n}\n\ncite {\n white-space: nowrap;\n color: black; /* slight contrast with #404040 of regular text */\n font-family: Consolas, \"Andale Mono WT\", \"Andale Mono\", \"Lucida Console\", \"Lucida Sans Typewriter\",\n \"DejaVu Sans Mono\", \"Bitstream Vera Sans Mono\", \"Liberation Mono\", \"Nimbus Mono L\", Monaco, \"Courier New\", Courier, monospace;\n font-style: normal;\n text-decoration: underline;\n}\n\n.borg-common-opt-ref {\n font-weight: bold;\n}\n\n.sidebar-toc ul li.toctree-l2 a,\n.sidebar-toc ul li.toctree-l3 a {\n padding-right: 25px;\n}\n\n#common-options .option {\n white-space: nowrap;\n}\n/* Remove the right-column max-width cap so content fills the full available width. */\n#right-column {\n max-width: none;\n}\n/* Hide the default RTD flyout since we show the version selector in the sidebar. */\nreadthedocs-flyout {\n display: none !important;\n}\n/* Version selector in the sidebar. */\n.version-selector {\n padding: 0 22px;\n margin: 7px 0 7px 0;\n font-size: 14px;\n}\n.version-selector label {\n display: block;\n margin-bottom: 4px;\n color: #000;\n}\n.version-selector select {\n width: 100%;\n padding: 4px;\n background-color: #fafafa;\n color: #000;\n border: 1px solid #ccc;\n border-radius: 3px;\n}\n.version-selector::after {\n content: '';\n display: block;\n border-top: 1px solid #ccc;\n margin: 7px 0 0 0;\n}\n/* Reduce top and bottom margin of searchbox block to 7px to match separator spacing. */\n.sidebar-block:has(#main-search) {\n margin-top: 7px;\n margin-bottom: 7px;\n}\n/* Reduce the separator margin below the search block to 7px. */\n.sphinxsidebar > .sidebar-block:has(#main-search):after {\n margin: 7px 22px 0 22px;\n}\n" }, { "path": "docs/changes.rst", "content": ".. _important_notes:\n\nImportant notes 2.x\n===================\n\nThis section provides information about security and corruption issues.\n\n(nothing to see here yet)\n\n.. _upgradenotes2:\n\nUpgrade Notes\n=============\n\nborg 1.2.x/1.4.x to borg 2.0\n----------------------------\n\nCompatibility notes:\n\n- This is a major \"breaking\" release that is not compatible with existing repositories.\n\n We tried to put all the necessary \"breaking\" changes into this release, so we\n hopefully do not need another breaking release in the near future. The changes\n were necessary for improved security, improved speed and parallelism,\n unblocking future improvements, getting rid of legacy crap and design\n limitations, having less and simpler code to maintain.\n\n You can use \"borg transfer\" to transfer archives from borg 1.2/1.4 repos to\n a new borg 2.0 repo, but it will need some time and space.\n\n Before using \"borg transfer\", you must have upgraded to borg >= 1.2.6 (or\n another borg version that was patched to fix CVE-2023-36811) and\n you must have followed the upgrade instructions at top of the change log\n relating to manifest and archive TAMs (borg2 just requires these TAMs now).\n\n- Command-line syntax was changed; scripts and wrappers will need changes:\n\n - You will usually either export BORG_REPO= into your environment or\n call borg like: \"borg -r \".\n In the docs, we usually omit \"-r ...\" for brevity.\n - The scp-style REPO syntax was removed; please use ssh://..., #6697\n - ssh:// URLs: Removed support for /~otheruser/, /~/ and /./, #6855.\n New format:\n\n - ssh://user@host:port/relative/path\n - ssh://user@host:port//absolute/path\n - -P / --prefix option was removed; please use the similar -a / --match-archives.\n - Archive names don't need to be unique anymore. To the contrary:\n It is now strongly recommended to use the identical name for borg create\n within the same series of archives to make borg work more efficiently.\n The name now identifies a series of archives; to identify a single archive,\n please use aid:, e.g., borg delete aid:d34db33f\n - In case you do NOT want to adopt the \"series name\" way of naming archives\n (like \"myarchive\") as we recommend, but keep using always-changing names\n (like \"myserver-myarchive-20241231\"), you can do that, but then you must\n make use of BORG_FILES_CACHE_SUFFIX and either set it to a constant suffix\n (like \"all\") or to a unique suffix per archive series (like\n \"myserver-myarchive\") so that borg can find the correct files cache.\n For the \"all\" variant, you must also set BORG_FILES_CACHE_TTL to a value\n greater than the count of different archives series you write to that repo.\n Usually borg uses a different files cache suffix per archive (series) name\n and defaults to BORG_FILES_CACHE_TTL=2 because that is sufficient for that.\n - The archive ID is always given separately from the repository.\n Unlike in borg 1.x, you must not give repo::archive.\n - The series name or archive ID is either given as a positional parameter,\n like:\n\n - borg create documents ~/Documents\n - borg diff aid:deadbeef aid:d34db33f\n - or, if the command makes sense for an arbitrary amount of archives, archives\n can be selected using a glob pattern, like:\n\n - borg delete -a 'sh:myarchive-2024-??-??'\n - borg recreate -a 'sh:myarchive-2024-??-??'\n - some borg 1.x commands that supported working on a repo AND on an archive\n were split into 2 commands, some others were renamed:\n\n - borg 2 repo commands:\n\n - borg repo-create # was: borg init\n - borg repo-list\n - borg repo-info\n - borg repo-delete\n - borg repo-compress\n - borg repo-space\n - borg 2 archive commands:\n\n - borg create NAME ...\n - borg list ID\n - borg extract ID ...\n - borg diff ID1 ID2\n - borg rename ID NEWNAME\n - borg info ID\n - borg delete ID\n - borg recreate ID ...\n - borg mount -a ID mountpoint ...\n\n For more details, please consult the docs or --help option output.\n - create/recreate/import-tar --timestamp: defaults to local timezone\n now (was: UTC)\n- some deprecated options were removed:\n\n - removed --remote-ratelimit (use --upload-ratelimit)\n - removed --numeric-owner (use --numeric-ids)\n - removed --nobsdflags (use --noflags)\n - removed --noatime (default now, see also --atime)\n - removed --save-space option (does not change behaviour)\n- removed --bypass-lock option\n- removed borg config command (only worked locally anyway)\n- compact command now requires access to the borg key if the repo is encrypted\n or authenticated\n- using --list together with --progress is now disallowed (except with --log-json), #7219\n- the --glob-archives option was renamed to --match-archives (the short option\n name -a is unchanged) and extended to support different pattern styles:\n\n - id: for identical string match (this is the new default!)\n - sh: for shell pattern / globbing match (this was used by --glob-archives)\n - re: for regular expression match\n\n So you might need to edit your scripts like e.g.::\n\n borg 1.x: --glob-archives 'myserver-2024-*'\n borg 2.0: --match-archives 'sh:myserver-2024-*'\n\n- use platformdirs 3.x.x instead of home-grown code. Due to that:\n\n - XDG_*_HOME is not honoured on macOS and on Windows.\n - BORG_BASE_DIR can still be used to enforce some base dir + .config/ or .cache/.\n - on macOS, the default directories move to native locations:\n config/data: ``~/Library/Application Support/borg/``,\n cache: ``~/Library/Caches/borg/``,\n runtime: ``~/Library/Caches/TemporaryItems/borg/``.\n - on Windows, the default directories are:\n config: ``C:\\Users\\\\AppData\\Roaming\\borg``,\n cache: ``C:\\Users\\\\AppData\\Local\\borg\\Cache``,\n data: ``C:\\Users\\\\AppData\\Local\\borg``.\n - **keyfile users on macOS (and Windows)**: borg 2 will look for key files in the\n new platform-specific config directory instead of ``~/.config/borg/keys/`` where\n borg 1.x stored them. You can set ``BORG_KEYS_DIR`` to point to the old location,\n or copy the key file to the new location. See :ref:`env_vars` and the ``borg transfer``\n documentation for details.\n- create: different included/excluded status chars, #7321\n\n - dry-run: now uses \"+\" (was: \"-\") and \"-\" (was: \"x\") for included/excluded status\n - non-dry-run: now uses \"-\" (was: \"x\") for excluded files\n\n Option --filter=... might need an update, if you filter for the status chars\n that were changed.\n- borg is now more strict and disallows giving some options multiple times -\n if that makes no sense. Highlander options, see #6269. That might make scripts\n fail now that somehow \"worked\" before (but maybe didn't work as intended due to\n the contradicting options).\n\n.. _changelog:\n\nChange Log 2.x\n==============\n\nVersion 2.0.0b21 (2026-03-16)\n-----------------------------\n\nPlease note:\n\nBeta releases are only for testing on NEW repos - do not use for production.\n\nFor upgrade and compatibility hints, please also read the section \"Upgrade Notes\"\nabove.\n\nNew features:\n\n- support https/http (REST) repositories via borgstore, #9480\n- use jsonargparse as CLI argument/option parser, also supporting YAML configs\n for defaults and auto-generated environment variables to override defaults, #6551\n- create --paths-from-shell-command, #5968\n- create: add --tags/--hostname/--username, #9401, #9402\n- create: implement \"file changed while backup\" detection on Windows, #9382\n- prune -v: now displays archive counts (total, kept, pruned), #9262\n- list --format: add fingerprint placeholder (fast!)\n- archive: use 3 timestamps (cleanly separate nominal archive timestamp from\n borg operation start/end info), #9400\n- benchmark crud: add --json-lines output option, #9165\n\nFixes:\n\n- prune: fix Archive.DoesNotExist when using --list, #9416\n- remove_dotdot_prefixes: remove bad assert, #9406\n- create --compress: expose Padmé size obfuscation (250) via CLI, #9286\n- remote: fix StoreObjectNotFound exception lost over RPC, #9380\n- passphrase: fail if multiple passphrase environment variables are set, #8834\n- cockpit: fix subprocess invocation in frozen binaries\n- cockpit: start the Borg runner after all widgets are mounted\n- debug format-obj: support all repository object types, #9391\n- benchmark crud: suppress compact warnings during benchmark runs, #9365\n- fix file: URL parsing for Windows\n\n - Linux: /abs/path -> file:///abs/path\n - Windows: c:/abs/path -> file:///c:/abs/path\n\nOther changes:\n\n- y2038: SUPPORT_32BIT_PLATFORMS = False, #9429.\n Not as bad as it sounds: 32bit platforms with 64bit time_t will still work.\n As of 2026, this is pretty much every platform that can run Borg reasonably well.\n- remove handwritten bash and zsh shell completions, #9178.\n these are now auto-generated via ``borg completion bash/zsh`` (using shtab).\n fish completions are kept until shtab gains fish support.\n- mount: warn about symlinks pointing outside of the mountpoint, #9254\n- use FILE_FLAG_WRITE_THROUGH on Windows for SyncFile data durability, #9388\n- extract: do not delete existing directory if possible, #4233\n- extract --continue: optimize processing of already existing dirs\n- mount: FUSE FS performance fix\n- prune: print hint to run compact to free space\n- prune: use same method to delete archives as delete subcommand, #9424\n- use xxhash from PyPI, #6535\n- use zstd from python lib or backports.zstd (python<'3.14'), #9261\n- swidth: use cross-platform implementation, #7493\n- platform: use F_FULLSYNC on macOS for SyncFile data durability, #9383\n- cache: add seek()/tell() to SyncFile, use SaveFile in _write_files_cache, #9390\n- cache: remove try_upgrade_to_b14() legacy migration, #9371\n- remove unnecessary checks: API_VERSION, check_python\n- time calculations: avoid floating point\n- Version: do not access private attributes, #9263\n- Windows platform (win32):\n\n - normalize drive letters, #9279\n - Path separator: internally always use \"/\", accept also \"\\\" for CLI arguments\n - map_chars: deal with invalid chars in paths on Windows\n- binary build:\n\n - use pyinstaller 6.18.0 for Python 3.14 compatibility\n - do not exclude ssl, needed for pyfuse3/trio, #9196\n - build with cockpit,s3,sftp extras installed, #9241\n - build Linux binaries with pyfuse3, #9196\n- Documentation:\n\n - jsonargparse: update docs about configs, auto-generated environment variables, precedence\n - fix S3 URL description, #9249\n - add a note that you need to install boto3 if you want to use S3/B2 URLs\n - rename BORG_RLIST_FORMAT to BORG_REPO_LIST_FORMAT, #9411\n - document platformdirs change and platform-specific directory paths, #7332\n - borgbackup.org: move RTD version selector to sidebar top-left, #8204\n - update SECURITY.md version table, #9346\n - fuse: add thread/async safety warning\n - upgrade http:// URLs to https:// and remove dead librelist.com link, #9342, #9302\n - man pages: fix broken :ref: references (e.g. borg_patterns), #7239\n - update deprecated pypi.python.org URLs to pypi.org, #9337\n - consolidate key backup info in borg key export, #6204\n - fix typos found by codespell, #9295\n - GitHub: enhance pull request template, #9334\n - archive specification, FAQ, #9248, #9053\n- testing / CI:\n\n - scripts/linux-run: run commands (e.g. tox) in a podman linux container,\n very useful when developing on macOS to test under Linux.\n - CI: add testing on omniOS (\"OpenSolaris\")\n - CI: use OpenBSD 7.8\n - CI: fix and re-enable Windows testing\n - CI: faster with borg-dir/borg.exe, #9236\n - add dependabot, #9308, #9349\n - add top-level permissions for least-privilege security, #9344\n - completion: focused tests for auto-generated shell completions\n (syntax validation, size sanity, borg-specific preamble behavior)\n - Speed up benchmark CPU tests with _BORG_BENCHMARK_CPU_TEST env var, #9414\n - fix mismatch in xattr test, #9238\n - xattr: document fakeroot xattr as Linux-only, add missing fakeroot skipping on FreeBSD, #9394\n - testsuite: remove deprecated manual cleanup in create_cmd_test\n - add borg.exe to PATH\n - fix tmpdir check on netbsd\n - enable Codecov Test Analytics, upgrade to codecov-action@v5\n - codecov: nothing to do for mypy and docs envs\n - add missing timeout-minutes to codeql, backport, and lint workflows, #9298\n - add path filters to lint and codeql workflows, #9328\n - cache tox environments\n - remove redundant tox runs, parallelize better, avoid unnecessary steps\n - add concurrency groups to cancel stale workflow runs, #9310\n - improve collecting coverage information, improve coverage, #9448\n - use locked requirements, add canary job, #9361\n - fix spurious sparse test fail on win32, #7616\n - fix race condition in test_with_lock, #8810\n - speed up prune/list/repo-list tests, #9324\n - add test for cockpit feature\n - add a borg create/extract timestamp test for y2261.\n\n\nVersion 2.0.0b20 (2025-12-24)\n-----------------------------\n\nNew features:\n\n- fat binary builds on GitHub (see assets on the GitHub releases page):\n\n - for Linux with glibc 2.35+ (Intel/AMD and ARM64)\n - for macOS 15+ (Apple Silicon/ARM64 and Intel)\n - using GitHub artifact attestations for release binaries, #9134\n- borg --cockpit: show status display based on Textual\n- Linux ACLs: use acl_to_any_text to avoid libacl name lookups, #8753.\n- export-tar/import-tar: support for POSIX ACLs (PAX format)\n- NetBSD: xattr support, #1332\n- mount: alternatively, work with high-level fuse library \"mfusepy\", which\n supports fuse 2 and 3, #9194. Try it with: pip install borg[mfusepy]\n- diff: --sort-by=field[,field,...], #8998\n- list --format: add \"inode\" placeholder\n- info: show cwd at the time of backup creation, #6191\n- improved tty-less progress reporting (--progress), #9055\n- BORG_MSGPACK_VERSION_CHECK=no to optionally disable the msgpack version\n check; default is \"yes\", use at your own risk, #9109.\n- completion: generate completion scripts for supported shells, #9172,\n uses shtab, supports bash and zsh.\n\nFixes:\n\n- extract: fs flags: use get/set to influence only specific flags, #9039, Linux, FreeBSD, macOS\n- transfer: fix borg transfer corrupting the source repo index, #9022\n- transfer: create a chunks list entry for missing chunks, see #9208\n- transfer: fix AttributeError with --dry-run, see #9199\n- old archives might not have a comment in metadata, see #9208\n- HardLinkManager: allow NoneType for contentless hardlinks, see #9208\n- legacyrepository: remove auto_recover, #9022\n- legacyremote: accept raise_missing in get/get_many to avoid TypeError\n with callers that pass it; no behavior change on legacy protocol, #9199\n- fix reading borg 1.x repo index, #9022\n- enable S3/B2 support of borgstore\n- mount --show-rc: display main process return code (rc), #8308\n- create: add exception handler for NODUMP-excluded directories, #9032\n- json: include archive keys in JSON lines when requested via --format, #9095\n- ensure valid file URLs are created from Windows paths\n- Windows: add missing guards around `preexec_fn=ignore_sigint`\n- preprocess_args: fix option name matching\n\nOther changes:\n\n- support Python 3.14, msgpack 1.1.2, use Cython 3.2.3\n- require setuptools>=78.1.1, #9042\n- \"bsdflags\" set_flags: remove compression flag support (did not work anyway)\n- Brewfile: use openssl@3\n- buzhash/buzhash64: initialise all-zero memory more efficiently\n- tests:\n\n - add fuzzing tests for chunkers\n - add tests for diff output of archives with hard links\n - read_only CM: skip test if cmd_immutable is unsuccessful, fixes #9021\n - save space in test_create_* tests\n - CI/tests: add SFTP/rclone/S3 repo testing\n - CI: add local servers for S3 and SFTP testing\n - CI: add misc. BSDs and Haiku OS (on GitHub Actions)\n - CI: do dynamic code analysis, #6819\n - transfer: add test for unexpected src repo index change, #9022\n - pyproject.toml: correctly define test environments for FUSE testing\n - add granularity_sleep, #9150\n - use context manager when opening files in patterns_test\n - FUSE related fixes/improvements, #9182\n - fix pynacl/libsodium build on freebsd, #9214\n - filter_xattrs now also filters some macOS xattrs\n - transfer: add --dry-run test\n - refactor id <-> name lookup for monkeypatching\n - CI: netbsd: enable xattrs on TMPDIR\n - improve fs cleanup directly after tests\n- Vagrant:\n\n - add Debian testing/Trixie box\n - add an OpenBSD 7.7 box\n - fix OpenIndiana box\n - drop macOS 10.12 box (binaries are built on GitHub now)\n - use Python 3.13.8 for binary building and tests\n - use PyInstaller 6.14.2 for binary building\n- docs:\n\n - update README for binaries\n - improve borg help patterns, #7144\n - patterns: clarify scope of default pattern style, #9004\n - extract: document how to use wildcards in PATHs, #8589\n - how to debug borg mount, #5461\n - document what happens when a new keyfile repo is created at the same path, #6230\n - update install docs to include `SETUPTOOLS_SCM_PRETEND_VERSION`\n - highlight archive series naming for fast incrementals, #8955\n - add Arch Linux to the 'Installing from source' docs\n - add systemd-inhibit and examples, #8989\n - code/docs: fix typos and grammar\n - some fixes/updates to the FAQ\n\n\nVersion 2.0.0b19 (2025-07-02)\n-----------------------------\n\nFixes:\n\n- reader: fix corruption issue \"forgetting\" all-zero bytestrings, #8963\n- import-tar: normalize the tarinfo name/linkname when used as hlm key.\n also: when printing the path, use the already normalized item.path.\n- import-tar: fix the dotslash issue, add test\n\nNew features:\n\n- create --files-changed=MODE option, #8958.\n control how borg detects whether a file has changed while it was backed up,\n valid modes are ctime (default), mtime (2nd best) or disabled (not recommended).\n\nOther changes:\n\n- to_key_filename: raise length limit to 120, #8966.\n This works around a test failure on systems with deep build directories.\n\n\nVersion 2.0.0b18 (2025-06-19)\n-----------------------------\n\nNew features:\n\n- experimental new \"buzhash64\" chunker (later, after testing, this shall become\n the default chunker in borg2):\n\n - add own cryptographically secure pseudo-random number generator (CSPRNG)\n based on AES256-CTR to create deterministic random, based on a 256bit seed.\n - use that to deterministically create a perfectly balanced buzhash64 table.\n - \"buzhash64\" chunker computes 64bit hash values for the chunking decision.\n - performance is similar to \"buzhash\" (measured on Apple M3P cpu).\n\n That should also resolve these points of criticism about the old \"buzhash\"\n 32bit code:\n\n - table_base: that the bits are not randomly distributed enough\n - that an XORed seed cancels out for specific window sizes\n - that XORing the table with a seed is equivalent to XORing the computed hash\n value with another constant\n\n Please test the chunkers extensively (e.g. with borg create, borg transfer),\n we can hardly change them \"in production\", because chunking differently also\n means not deduplicating with old chunks. So, in case there are changes\n needed, we need to find and fix them now while borg is in beta.\n\n See also some other chunker changes listed below \"Other changes\".\n- serve: add --permissions option as an alternative to BORG_REPO_PERMISSIONS env var\n- create: auto-exclude items based on xattrs or NODUMP, see #4972\n\n no options yet, just hardcoded macOS and Linux xattrs.\n removed the --exclude-nodump option, it is also done automagically now.\n\n also: create: read stat attrs, xattrs, ACLs early, before file contents.\n\nFixes:\n\n- compact: fix cleaning archives directory (catch correct exception, use\n logger.warning, improve error msg)\n\nOther changes:\n\n- support Python 3.14\n- msgpack: allow 1.1.1, version check: ignore \"rc\" or other version elements\n- add derive_key to derive new keys from existing key material\n- refactor the chunkers, #8882 #8883:\n\n - transform buzhash chunker C code to Cython\n - split concerns into FileFMAPReader, FileReader, Chunker*:\n\n - FileFMAPReader reads blocks from the input file, supporting sparse\n files and fmaps.\n - FileReader uses FileFMAPReader to fill its buffer and offers clients a\n `.read(size)` method so they can read pieces of the data.\n - all chunkers now use the FileReader/FileFMAPReader code\n - split code and test module into packages\n- \"fixed\" chunker: add fixed chunker tests to selftest\n- \"fixed\" chunker: do not assert on short header read\n- \"buzhash*\" chunker: use safe_fadvise\n- \"buzhash\" chunker: reject even window size, #8868\n- fish: fix archive name completion\n- refactor: modularize tests\n- refactor: use pathlib.Path\n- tests / CI:\n\n - CI: add bandit, a security-oriented static analysis tool\n - CI: disable windows as the file:// repo URLs are still broken on windows.\n - tests: tox: use native pyproject.toml configuration\n - more chunker-related tests\n- docs:\n\n - add docs for serve --permissions / BORG_REPO_PERMISSIONS\n - borg-serve: simplify example of env in authorized_keys, #8318\n - fix mistyped CVE number\n\n\nVersion 2.0.0b17 (2025-05-23)\n-----------------------------\n\nNew features:\n\n- transfer: implement --chunker-params to re-chunk while transferring, #8706\n- list --depth=N: list files up to N depth in path hierarchy, #8268\n- compact: also clean up files cache, #8852\n- `BORG_REPO_PERMISSIONS=all|no-delete|write-only|read-only`, #8823\n\n The posixfs borgstore backend implements permissions to make\n testing with differently permissive stores easier.\n\n The env var selects from pre-defined permission configurations\n within borg and gives the chosen permissions config to borgstore.\n borg uses borgstore's posixfs backend only for file: and ssh: repos.\n\nFixes:\n\n- correct the signature of __set_name__ as cython 3.1 added support,\n fixing build on Cython 3.1, #6858\n- compact/check: fix bug not writing the complete index, #8813\n- compact: add --iec option, #8831\n- check/compact/analyze: show archive timestamp in local tz, #8814\n- repo-space: enable ssh: repo testing, fix AttributeError, #8815\n- repo-info: fix output formatting\n\nOther changes:\n\n- require borgstore 0.3.x\n- some updates and fixes for shell completions, needs more work\n- dir_is_tagged/_is_cachedir: add fd-based operations\n- cython: suppress compiler warning about CYTHON_FALLTHROUGH in unreachable code\n- source code: `pyupgrade --py310-plus ./**/*.py`\n- tests:\n\n - add/improve tests for repo-compress --stats, transfer, repo-space\n - split helpers tests from a single module into borg.testsuite.helpers package\n - save temp space (good for ramdisk users)\n - fix diff cmd test on macOS HFS+, #8860\n - test validity of shell completion files\n - CI: fix and enable windows CI, #8728\n - CI: upload coverage for windows tests\n - CI: install zsh and fish so we can test shell completions\n- docs:\n\n - must have the release tags in the local repo, #8582\n - remove outdated docs/man files about borg change-passphrase\n - add S3/B2 urls to documentation for repository urls, #8833\n\n\nVersion 2.0.0b16 (2025-05-06)\n-----------------------------\n\nFixes:\n\n- chunks cache: invalidate old chunk index cache, #8795\n- compact: always write updated chunkindex to repo, #8791\n- ChunksMixin: don't use self._chunks until it is demand-built, #8785\n- AdhocWithFilesCache: fix call to _maybe_write_chunks_cache\n- format_time: output date/time in local tz, #8802\n- check: ask for key passphrase early, #1931\n- only obfuscate the size of file content chunks, #7559\n- better support other repo by misc. passphrase env vars, #8457\n\n - passphrases now come from `BORG_[OTHER_]PASSPHRASE`, `BORG_[OTHER_]PASSCOMMAND`\n or `BORG_[OTHER_]PASSPHRASE_FD`.\n - `borg repo-create --repo B --other-repo A` does not silently copy the\n passphrase of key A to key B anymore, but either asks for the passphrase\n or reads it from env vars.\n\nOther changes:\n\n- remove support for / testing on Python 3.9\n- docs: borg serve --repo is not supported, #8591\n- remove remainders of append-only and quota support\n- remove cygwin < 2.8.0 bug workaround\n- fix remote api versioning\n\n\nVersion 2.0.0b15 (2025-04-22)\n-----------------------------\n\nNew features:\n\n- compact: without --stats, it will be faster by using the cached chunks index.\n with --stats it will be as slow as before, listing all repo objs.\n- compact: support --dry-run (do nothing), #8300\n- extract: --dry-run now displays +/- status flags (included/excluded), #8564\n- allow timespan to be specified with common time units, #8624\n- enhance passphrase handling, #8496.\n\n Setting `BORG_DEBUG_PASSPHRASE=YES` enables passphrase debug logging to\n stderr, showing passphrase, hex utf-8 byte sequence and related env vars if\n a wrong passphrase was encountered.\n\n Setting `BORG_DISPLAY_PASSPHRASE=YES` now always shows passphrase and its hex\n utf-8 byte sequence.\n- add {unixtime} placeholder, #8522\n- implement padme chunk size obfuscation (SPEC 250), #8705\n- macOS: retrieve birthtime in nanosecond precision via system call, #8724\n\nBug fixes:\n\n- borg exits when assertions are disabled with Python optimizations, #8649\n- yes(): deal with UnicodeDecodeError in input(), #6984\n- fix remote repository exception handling / modern exit codes, #8631\n- freebsd: fix nfs4 acl processing, #8756.\n This issue only affected borg extract --numeric-ids when processing NFS4\n ACLs, it didn't affect POSIX ACL processing.\n\nOther changes:\n\n- adapt to and require borghash 0.1.0\n- adapt to and require borgstore 0.2.0 (new s3/b2 backend, fixes/improvements)\n- create: remove --make-parent-dirs option (borgstore now does this automatically), #8619\n- iter_items: decouple item iteration and content data chunks preloading\n- remote: simplify code, add debug logging\n- pyproject.toml: SPDX expression for license, add license-files, #8771\n- Item: remove .chunks_healthy, #8559\n- OpenBSD fixes:\n\n - support other OpenSSL versions on OpenBSD, #8553\n - vagrant: fix OpenBSD box, #8506\n - Filter test output with LibreSSL related warnings on OpenBSD\n- macOS: fix brew's broken pkg-config -> pkgconf transition\n- tests: ignore 'com.apple.provenance' xattr (macOS specific)\n- vagrant updates:\n\n - use pyinstaller 6.11.1 (also use this in msys2 build scripts)\n - use python 3.12.10\n - build binaries with borgstore[sftp], #8574\n- docs:\n\n - automated backup: append to SYSTEMD_WANTS rather than overwrite, #8641\n - fix udev rule priority in automated-local.rst, #8639\n - FAQ: Why backups are slow on a Linux server that is a member of a windows domain? #8636\n - within a shell, cli options with special characters may require quoting, #8578\n - update prune documentation for new --keep-within intervals, #8630\n - borg serve: recommend using a simple shell, #3818\n - update install docs (requirements, pkgconfig, fuse), #8342\n - libffi-dev is required for argon2-cffi-bindings\n - add undelete command to index\n - borg commands updated with --repo option, #8550\n - FAQ: add entry about pure-python msgpack warning, #8323\n - readthedocs theme fixes\n\n - bring back highlighted content preview in search results.\n - fix erroneous warning about missing javascript support.\n\n\nVersion 2.0.0b14 (2024-11-17)\n-----------------------------\n\nNew features:\n\n- delete: now only soft-deletes archives (same for prune)\n- repo-list: --deleted lists deleted archives\n- undelete: undelete soft-deleted archives, #8500\n\nFixes:\n\n- chunks index cache:\n\n - enable partial/incremental updates (F_NEW flag).\n - write chunks index every 10mins, #8503.\n this makes sure progress is not totally lost when a backup is interrupted.\n - write to repo/cache/chunks. to enable parallel updates.\n- mount: fix check_pending_archive to give correct root dir, #8528\n\nOther changes:\n\n- repo-compress: reduce memory consumption (F_COMPRESS flag)\n- files cache: reduce memory consumption, #5756\n- check: rename --undelete-archives to --find-lost-archives\n- check: rebuild_archives_directory: accelerate by only reading metadata\n- shell completions: adapt zsh for borg 2.0.0b13 - needs more work!\n- chunk index: rename .refcount to .flags, use it for user and system flags.\n- vagrant:\n\n - add bookworm32 box for 32bit platform testing\n - fix pythons on freebsd14\n - simplify openindiana box setup\n- docs:\n\n - remove --bypass-lock, small changes regarding compression\n - FAQ: clean up entries regarding SSH settings\n\n\nVersion 2.0.0b13 (2024-10-31)\n-----------------------------\n\nNew features:\n\n- implement special tags, @PROT tag for protecting archives, #953.\n\n borg won't delete/prune/recreate protected archives.\n- prune: add quarterly pruning strategy, #8337.\n- import-tar/export-tar: add xattr support for PAX format, #2521.\n\nFixes:\n\n- simple error msgs for existing / non-existing repo, no tracebacks, #8475.\n- mount: create unique directory names, #8461.\n- diff: suppress modified changes for files which weren't actually modified.\n- diff: do not test for ctime difference on windows.\n- prune: fix exception when NAME is given, #8486\n- repo-create: build and cache an empty ChunkIndex.\n- work around missing size/nfiles archive metadata, #8491\n- lock after checking repo exists, #8485\n\nOther changes:\n\n- new file:, rclone:, ssh:, sftp: URLs, #8372, #8446.\n\n new way to deal with absolute vs. relative paths.\n- require borgstore ~= 0.1.0, require borghash ~= 0.0.1.\n- new hashtable code based on borghash project:\n\n - borghash replaces old / hard to maintain _hashindex.c code.\n - implement ChunkIndex, NSIndex1, FuseVersionsIndex using borghash.HashTableNT.\n - rewrite NSIndex1 (borg 1.x) on-disk format read/write methods in Cython.\n - remove NSIndex (early borg2) data structure / serialization code for repo index.\n - change xxh64 seed for ChunkIndex to invalidate old cache contents.\n - chunks index: show hashtable stats at debug log level, #506.\n- check (repository part): build and cache a ChunkIndex.\n\n check (archives part): use cached ChunkIndex from check (repository part).\n- export-tar: switch default to PAX format.\n- docs:\n\n - update URL docs\n - mount: document on-demand loading, perf tips, #7173.\n - borg/borgfs detects internally under which name it was invoked, #8207.\n - better link modern return codes, #8370.\n - binary: using the directory build is faster, #8008.\n - update \"Running the tests (using the pypi package)\", #6386.\n- github CI:\n\n - temporarily disabled windows CI, #8474.\n - msys2: use pyinstaller 6.10.0.\n - msys2: install rclone.\n- tests:\n\n - rename test files so that pytest default discovery finds them.\n - call register_assert_rewrite before importing borg.testsuite.\n - move conftest.py one directory level higher.\n - remove hashindex tests from selftests (borghash project has own tests).\n\n\nVersion 2.0.0b12 (2024-10-03)\n-----------------------------\n\nNew features:\n\n- tag: new command to set, add, remove tags.\n- repo-list: add tags/hostname/username/comment to default format, reorder, adjust.\n\n Idea: not putting these into the archive name, but keeping them separate.\n- repo-list --short: only print archive IDs (unique IDs, used for scripting).\n- implement --match-archives user:USERNAME host:HOSTNAME tags:TAG1,TAG2,...\n- allow -a / --match-archives multiple times (logical AND).\n\n E.g.: borg delete -a home -a user:kenny -a host:kenny-pc\n- analyze: list changed chunks' sizes per directory.\n\nFixes:\n\n- locking: also refresh the lock in other repo methods. avoid repo lock\n getting stale when processing lots of unchanged files, #8442.\n- make sure the store gets closed in case of exceptions, #8413.\n- msgpack: increase max_buffer_size to ~4GiB, #8440.\n- Location.canonical_path: fix protocol and host display, #8446.\n\nOther changes:\n\n- give borgstore.Store a complete levels configuration, #8432.\n- add BORG_STORE_DATA_LEVELS=2 env var.\n- check: also display archive timestamp.\n- vagrant:\n\n - use python 3.12.6 for binary builds.\n - new testing box based on bento/ubuntu-24.04.\n - install Rust on BSD.\n\n\nVersion 2.0.0b11 (2024-09-26)\n-----------------------------\n\nNew features:\n\n- Support rclone:// URLs for borg repositories.\n\n This enables 70+ cloud storage products, including Amazon S3, Backblaze B2,\n Ceph, Dropbox, ftp(s), Google Cloud Storage, Google Drive, Microsoft Azure,\n Microsoft OneDrive, OpenStack Swift, pCloud, Seafile, sftp, SMB / CIFS and\n WebDAV!\n\n See https://rclone.org/ for more details.\n- Parallel operations in same repo from same client (same user/machine).\n- Archive series feature, #7930.\n\n TL;DR: a NAME now identifies a series of identically named archives,\n to identify a specific single archive, use aid:.\n\n in borg 1.x, we used to put a timestamp into the archive name, because borg1\n required unique archive names.\n\n borg2 does not require unique archive names, but it encourages you to even\n use a identical archive names within the same SERIES of archives, e.g. you\n could backup user files to archives named \"user-files\" and system files to\n archives named \"system-files\".\n that makes matching (e.g. for prune, for the files cache, ...) much simpler\n and borg now KNOWS which archives belong to the same series (because they all\n have the same name).\n- info/delete/prune: allow positional NAME argument, e.g.:\n\n - borg prune --keep-daily 30 \n - borg delete aid:\n- create: also archive inode number, #8362\n\n Borg can use this when using archive series to rebuild the local files cache\n from the previous archive (of the same series) in the repository.\n\nFixes:\n\n- Remove superfluous repository.list() call. for high latency repos\n (like sftp, cloud), this improves performance of borg check and compact.\n- repository.list: refresh lock more frequently\n- misc. commands fixed for non-unique archive names\n- remote: allow get_manifest method\n- files cache: fix rare race condition with data loss potential, #3536\n- storelocking: misc. fixes / cleanups\n\nOther changes:\n\n- Cache the chunks index in the repository, #8397.\n Improves high latency repo performance for most commands compared to b10.\n- repo-compress: faster by using chunks index rather than repository.list().\n- Files cache entries now have both ctime AND mtime.\n- Borg updates the ctime and mtime of known and \"unchanged\" files, #4915.\n- Rebuild files cache from previous archive in same series, #8385.\n- Reduce RAM usage by splitting the files cache by archive series, #5658.\n- Remove AdHocCache, remove BORG_CACHE_IMPL (we only have one implementation).\n- Docs: user@ and :port are optional in sftp and ssh URLs.\n- CI: re-enable windows build after fixing it.\n- Upgrade pyinstaller to 6.10.0.\n- Increase IDS_PER_CHUNK, #6945.\n\n\nVersion 2.0.0b10 (2024-09-09)\n-----------------------------\n\nNew features:\n\n- borgstore based repository, file:, ssh: and sftp: for now, more possible.\n- repository stores objects separately now, not using segment files.\n this has more fs overhead, but needs much less I/O because no segment\n files compaction is required anymore. also, no repository index is\n needed anymore because we can directly find the objects by their ID.\n- locking: new borgstore based repository locking with automatic stale\n lock removal (if lock does not get refreshed, if lock owner process is dead).\n- simultaneous repository access for many borg commands except check/compact.\n the cache lock for adhocwithfiles is still exclusive though, so use\n BORG_CACHE_IMPL=adhoc if you want to try that out using only 1 machine\n and 1 user (that implementation doesn't use a cache lock). When using\n multiple client machines or users, it also works with the default cache.\n- delete/prune: much quicker now and can be undone.\n- check --repair --undelete-archives: bring archives back from the dead.\n- repo-space: manage reserved space in repository (avoid dead-end situation if\n repository filesystem runs full).\n\nBugs/issues fixed:\n\n- a lot! all linked from PR #8332.\n\nOther changes:\n\n- repository: remove transactions, solved differently and much simpler now\n (convergence and write order primarily).\n- repository: replaced precise reference counting with \"object exists in repo?\"\n and \"garbage collection of unused objects\".\n- cache: remove transactions, remove chunks cache.\n removed LocalCache, BORG_CACHE_IMPL=local, solving all related issues.\n as in beta 9, adhowwithfiles is the default implementation.\n- compact: needs the borg key now (run it clientside), -v gives nice stats.\n- transfer: archive transfers from borg 1.x need the --from-borg1 option\n- check: reimplemented / bigger changes.\n- code: got rid of a metric ton of not needed complexity.\n when borg does not need to read borg 1.x repos/archives anymore, after\n users have transferred their archives, even much more can be removed.\n- docs: updated / removed outdated stuff\n- renamed r* commands to repo-*\n\n\nVersion 2.0.0b9 (2024-07-20)\n----------------------------\n\nNew features:\n\n- add BORG_CACHE_IMPL, default is \"adhocwithfiles\" to test the new cache\n implementation, featuring an adhoc non-persistent chunks cache and a\n persistent files cache. See the docs for other values.\n\n Requires to run \"borg check --repair --archives-only\" to delete orphaned\n chunks before running \"borg compact\" to free space! These orphans are\n expected due to the simplified refcounting with the AdHocFilesCache.\n- make BORG_EXIT_CODES=\"modern\" the default, #8110\n- add BORG_USE_CHUNKS_ARCHIVE env var, #8280\n- automatically rebuild cache on exception, #5213\n\nBug fixes:\n\n- fix Ctrl-C / SIGINT behaviour for pyinstaller-made binaries, #8155\n- delete: fix error handling with Ctrl-C\n- rcompress: fix error handling with Ctrl-C\n- delete: fix error handling when no archive is specified, #8256\n- setup.py: fix import error reporting for cythonize import, see #8208\n- create: deal with EBUSY, #8123\n- benchmark: inherit options --rsh --remote-path, #8099\n- benchmark: fix return value, #8113\n- key export: fix crash when no path is given, fix exception handling\n\nOther changes:\n\n- setup.py: detect noexec build fs issue, see #8208\n- improve acl_get / acl_set error handling (forward port from 1.4-maint)\n- allow msgpack 1.1.0\n- vagrant: use pyinstaller 6.7.0\n- use Python 3.11.9 for binary builds\n- require Cython 3.0.3 at least, #8133\n- docs: add non-root deployment strategy\n\n\nVersion 2.0.0b8 (2024-02-20)\n----------------------------\n\nNew features:\n\n- create: add the slashdot hack, update docs, #4685\n- BORG_EXIT_CODES=modern: optional more specific return codes (for errors and warnings).\n\n The default value of this new environment variable is \"legacy\", which should result in\n a behaviour similar to borg 1.2 and older (only using rc 0, 1 and 2).\n \"modern\" exit codes are much more specific (see the internals/frontends docs).\n- implement \"borg version\" (shows client and server version), #7829\n\nFixes:\n\n- docs: CVE-2023-36811 upgrade steps: consider checkpoint archives, #7802\n- check/compact: fix spurious reappearance of orphan chunks since borg 1.2, #6687 -\n this consists of 2 fixes:\n\n - for existing chunks: check --repair: recreate shadow index, #7897 #6687\n - for newly created chunks: update shadow index when doing a double-put, #7896 #5661\n\n If you have experienced issue #6687, you may want to run borg check --repair\n after upgrading to borg 1.2.7 to recreate the shadow index and get rid of the\n issue for existing chunks.\n- check: fix return code for index entry value discrepancies\n- LockRoster.modify: no KeyError if element was already gone, #7937\n- create --X-from-command: run subcommands with a clean environment, #7916\n- list --sort-by: support \"archive\" as alias of \"name\", #7873\n- fix rc and msg if arg parsing throws an exception, #7885\n- PATH: do not accept empty strings, #4221\n- fix invalid pattern argument error msg\n- zlib legacy decompress fixes, #7883\n\nOther changes:\n\n- replace archive/manifest TAMs by typed repo objects (ro_type), docs, #7670\n- crypto: use a one-step kdf for session keys, #7953\n- remove recreate --recompress option, use the more efficient repo-wide \"rcompress\".\n- include unistd.h in _chunker.c (fix for Python 3.13)\n- allow msgpack 1.0.7\n- allow platformdirs 4, #7950\n- use and require cython3\n- move conftest.py to src/borg/testsuite, #6386\n- use less setup.py, use pip and build\n- linux: use pkgconfig to find libacl\n- borg.logger: use same method params as python logging\n- create and use Brewfile, document \"brew bundle\" install (macOS)\n- blacken master branch\n- prevent CLI argument issues in scripts/glibc_check.py\n- pyproject.toml: exclude source files which have been compiled, #7828\n- sdist: dynamically compute readme (long_description)\n- init: better borg key export instructions\n- scripts/make.py: move clean, build_man, build_usage to there,\n so we do not need to invoke setup.py directly, update docs\n- vagrant:\n\n - use openssl 3.0 on macOS\n - add script for fetching borg binaries from VMs, #7989\n - use generic/openbsd7 box\n - netbsd: test on py311 only\n - remove debian 9 \"stretch\" box\n - use freebsd 14, #6871\n - use python 3.9.4 for tests, latest python 3.11.7 for binary builds\n - use pyinstaller 6.3.0\n- docs:\n\n - add typical PR workflow to development docs, #7495\n - improve docs for borg with-lock, add example #8024\n - create disk/partition sector backup by disk serial number\n - Add \"check.rebuild_refcounts\" message\n - not only attack/unsafe, can also be a fs issue, #7853\n - use virtualenv on Cygwin\n - readthedocs: also build offline docs, #7835\n - do not refer to setup.py installation method\n - how to run the testsuite using the dist package\n - requirements are defined in pyproject.toml\n\n\nVersion 2.0.0b7 (2023-09-14)\n----------------------------\n\nNew features:\n\n- BORG_WORKAROUNDS=authenticated_no_key to extract from authenticated repos\n without having the borg key, #7700\n\nFixes:\n\n- archive tam verify security fix, fixes CVE-2023-36811\n- remote logging/progress: use callback to send queued records, #7662\n- make_path_safe: remove test for backslashes, #7651\n- benchmark cpu: use sanitized path, #7654\n- create: do not try to read parent dir of recursion root, #7746\n\nOther changes:\n\n- always implicitly require archive TAMs (all archives have TAMs since borg 1.2.6)\n- always implicitly require manifest TAMs (manifests have TAMs since borg 1.0.9)\n- rlist: remove support for {tam} placeholder, archives are now always TAM-authenticated.\n- support / test on Python 3.12\n- allow msgpack 1.0.6 (which has py312 wheels), #7810\n- manifest: move item_keys into config dict (manifest.version == 2 now), #7710\n- replace \"datetime.utcfromtimestamp\" to avoid deprecation warnings with Python 3.12\n- properly normalise paths on Windows (forward slashes, integrate drive letter into path)\n- Docs:\n\n - move upgrade / compat. notes to own section, see #7546\n - fix borg delete examples, #7759\n - improve rcreate / related repos docs\n - automated-local.rst: use UUID for consistent udev rule\n - rewrite `borg check` docs, #7578\n - misc. other docs updates\n- Tests / CI / Vagrant:\n\n - major testsuite refactoring: a lot more tests now use pytest, #7626\n - freebsd: add some ACL tests, #7745\n - fix test_disk_full, #7617\n - fix failing test_get_runtime_dir test on OpenBSD, #7719\n - CI: run on ubuntu 22.04\n - CI: test building the docs\n - simplify flake8 config, fix some complaints\n - use pyinstaller 5.13.1 to build the borg binaries\n\n\nVersion 2.0.0b6 (2023-06-11)\n----------------------------\n\nNew features:\n\n- diff: include changes in ctime and mtime, #7248\n- diff: sort JSON output alphabetically\n- diff --content-only: option added to ignore metadata changes\n- diff: add --format option, #4634\n- import-tar --ignore-zeros: new option to support importing concatenated tars, #7432\n- debug id-hash / parse-obj / format-obj: new debug commands, #7406\n- transfer --compression=C --recompress=M: recompress while transferring, #7529\n- extract --continue: continue a previously interrupted extraction, #1356\n- prune --list-kept/--list-pruned: only list the kept (or pruned) archives, #7511\n- prune --short/--format: enable users to format the list output, #3238\n- implement BORG__FORMAT env vars for prune, list, rlist, #5166\n- rlist: size and nfiles format keys\n- implement unix domain (ipc) socket support, #6183::\n\n borg serve --socket # server side (not started automatically!)\n borg -r socket:///path/to/repo ... # client side\n\n- add get_runtime_dir / BORG_RUNTIME_DIR (contains e.g. .sock and .pid file)\n- support shell-style alternatives, like: sh:image.{png,jpg}, #7602\n\nFixes:\n\n- do not retry on permission errors (pointless)\n- transfer: verify chunks we get using assert_id, #7383\n- fix config/cache dir compatibility issues, #7445\n- xattrs: fix namespace processing on FreeBSD, #6997\n- ProgressIndicatorPercent: fix space computation for wide chars, #3027\n- delete: remove --cache-only option, #7440.\n for deleting the cache only, use: borg rdelete --cache-only\n- borg debug get-obj/put-obj: fixed chunk id\n- create: ignore empty paths, print warning, #5637\n- extract: support extraction of atime/mtime on win32\n- benchmark crud: use TemporaryDirectory below given path, #4706\n- Ensure that cli options specified with action=Highlander can only be set once, even\n if the set value is a default value. Add tests for action=Highlander, #7500, #6269.\n- Fix argparse error messages from misc. validators (being more specific).\n- put security infos into data dir, add BORG_DATA_DIR env var, #5760\n- setup.cfg: remove setup_requires (we have a pyproject.toml for that), #7574\n- do not crash for empty archives list in borg rlist date based matching, #7522\n- sanitize paths during archive creation and extraction, #7108 #7099\n- make sure we do not get backslashes into item paths\n\nOther changes:\n\n- allow msgpack 1.0.5 also\n- development.lock.txt: upgrade cython to 0.29.35, misc. other upgrades\n- clarify platformdirs requirements, #7393.\n 3.0.0 is only required for macOS due to breaking changes.\n 2.6.0 was the last breaking change for Linux/UNIX.\n- mount: improve mountpoint error msgs, see #7496\n- more Highlander options, #6269\n- Windows: simplify building (just use pip)\n- refactor toplevel exception handling, #6018\n- remove nonce management, related repo methods (not needed for borg2)\n- borg.remote: remove support for borg < 1.1.0\n ($LOG, logging setup, exceptions, rpc tuple data format, version)\n- new remote and progress logging, #7604\n- borg.logger: add logging debugging functionality\n- add function to clear empty directories at end of compact process\n- unify scanning and listing of segment dirs / segment files, #7597\n- replace `LRUCache` internals with `OrderedDict`\n- docs:\n\n - add installation instructions for Windows\n - improve --one-file-system help and docs (macOS APFS), #5618 #4876\n - BORG_KEY_FILE: clarify docs, #7444\n - installation: add link to OS dependencies, #7356\n - update FAQ about locale/unicode issues, #6999\n - improve mount options rendering, #7359\n - make timestamps in manual pages reproducible.\n - describe performing pull-backups via ssh remote forwarding\n - suggest to use forced command when using remote-forwarding via ssh\n - fix some -a / --match-archives docs issues\n - incl./excl. options header, clarify --path-from-stdin exclusive control\n - add note about MAX_DATA_SIZE\n - update security support docs\n - improve patterns help\n\n- CI / tests / vagrant:\n\n - added pre-commit for linting purposes, #7476\n - resolved mode bug and added sleep clause for darwin systems, #7470\n - \"auto\" compressor tests: do not assume zlib is better than lz4, #7363\n - add stretch64 VM with deps built from source\n - misc. other CI / test fixes and updates\n - vagrant: add lunar64 VM, fix packages_netbsd\n - avoid long ids in pytest output\n - tox: package = editable-legacy, #7580\n - tox under fakeroot: fix finding setup_docs, #7391\n - check buzhash chunksize distribution, #7586\n - use debian/bookworm64 box\n\n\nVersion 2.0.0b5 (2023-02-27)\n----------------------------\n\nNew features:\n\n- create: implement retries for individual fs files\n (e.g. if a file changed while we read it, if a file had an OSError)\n- info: add used storage quota, #7121\n- transfer: support --progress\n- create/recreate/import-tar: add --checkpoint-volume option\n- support date-based matching for archive selection,\n add --newer/--older/--newest/--oldest options, #7062 #7296\n\nFixes:\n\n- disallow --list with --progress, #7219\n- create: fix --list --dry-run output for directories, #7209\n- do no assume hardlink_master=True if not present, #7175\n- fix item_ptrs orphaned chunks of checkpoint archives\n- avoid orphan content chunks on BackupOSError, #6709\n- transfer: fix bug in obfuscated data upgrade code\n- fs.py: fix bug in f-string (thanks mypy!)\n- recreate: when --target is given, do not detect \"nothing to do\", #7254\n- locking (win32): deal with os.rmdir/listdir PermissionErrors\n- locking: thread id must be parsed as hex from lock file name\n- extract: fix mtime when ResourceFork xattr is set (macOS specific), #7234\n- recreate: without --chunker-params borg shall not rechunk, #7336\n- allow mixing --progress and --list in log-json mode\n- add \"files changed while reading\" to Statistics class, #7354\n- fixed keys determination in Statistics.__add__(), #7355\n\nOther changes:\n\n- use local time / local timezone to output timestamps, #7283\n- update development.lock.txt, including a setuptools security fix, #7227\n- remove --save-space option (does not change behaviour)\n- remove part files from final archive\n- remove --consider-part-files, related stats code, update docs\n- transfer: drop part files\n- check: show id of orphaned chunks\n- ArchiveItem.cmdline list-of-str -> .command_line str, #7246\n- Item: symlinks: rename .source to .target, #7245\n- Item: make user/group/uid/gid optional\n- create: do not store user/group for stdin data by default, #7249\n- extract: chown only if we have u/g info in archived item, #7249\n- export-tar: for items w/o uid/gid, default to 0/0, #7249\n- fix some uid/gid lookup code / tests for win32\n- cache.py: be less verbose during cache sync\n- update bash completion script commands and options, #7273\n- require and use platformdirs 3.x.x package, tests\n- better included/excluded status chars, docs, #7321\n- undef NDEBUG for chunker and hashindex (make assert() work)\n- assert_id: better be paranoid (add back same crypto code as in old borg), #7362\n- check --verify_data: always decompress and call assert_id(), #7362\n- make hashindex_compact simpler and probably faster, minor fixes, cleanups, more tests\n- hashindex minor fixes, refactor, tweaks, tests\n- pyinstaller: remove icon\n- validation / placeholders / JSON:\n\n - implement (text|binary)_to_json: key (text), key_b64 (base64(binary))\n - remove bpath, barchive, bcomment placeholders / JSON keys\n - archive metadata: make sure hostname and username have no surrogate escapes\n - text attributes (like archive name, comment): validate more strictly, #2290\n - transfer: validate archive names and comment before transfer\n - json output: use text_to_json (path, target), #6151\n- docs:\n\n - docs and comments consistency, readability and spelling fixes\n - fix --progress display description, #7180\n - document how borg deals with non-unicode bytes in JSON output\n - document another way to get UTF-8 encoding on stdin/stdout/stderr, #2273\n - pruning interprets timestamps in the local timezone where borg prune runs\n - shellpattern: add license, use copyright/license markup\n - key change-passphrase: fix --encryption value in examples\n - remove BORG_LIBB2_PREFIX (not used any more)\n - Installation: Update Fedora in distribution list, #7357\n - add .readthedocs.yaml (use py311, use non-shallow clone)\n- tests:\n\n - fix archiver tests on Windows, add running the tests to Windows CI\n - fix tox4 passenv issue, #7199\n - github actions updates (fix deprecation warnings)\n - add tests for borg transfer/upgrade\n - fix test hanging reading FIFO when `borg create` failed\n - mypy inspired fixes / updates\n - fix prune tests, prune in localtime\n - do not look up uid 0 / gid 0, but current process uid/gid\n - safe_unlink tests: use os.link to support win32 also\n - fix test_size_on_disk_accurate for large st_blksize, #7250\n - relaxed timestamp comparisons, use same_ts_ns\n - add test for extracted directory mtime\n - use \"fail\" chunker to test erroneous input file skipping\n\n\nVersion 2.0.0b4 (2022-11-27)\n----------------------------\n\nFixes:\n\n- transfer/upgrade: fix borg < 1.2 chunker_params, #7079\n- transfer/upgrade: do not access Item._dict, #7077\n- transfer/upgrade: fix crash in borg transfer, #7156\n- archive.save(): always use metadata from stats, #7072\n- benchmark: fixed TypeError in compression benchmarks, #7075\n- fix repository.scan api minimum requirement\n- fix args.paths related argparsing, #6994\n\nOther changes:\n\n- tar_filter: recognize .tar.zst as zstd, #7093\n- adding performance statistics to borg create, #6991\n- docs: add rcompress to usage index\n- tests:\n\n - use github and MSYS2 for Windows CI, #7097\n - win32 and cygwin: test fixes / skip hanging test\n - vagrant / github CI: use python 3.11.0 / 3.10.8\n- vagrant:\n\n - upgrade pyinstaller to 5.6.2 (supports python 3.11)\n - use python 3.11 to build the borg binary\n\nVersion 2.0.0b3 (2022-10-02)\n----------------------------\n\nFixes:\n\n- transfer: fix user/group == None crash with borg1 archives\n- compressors: avoid memoryview related TypeError\n- check: fix uninitialised variable if repo is completely empty, #7034\n- do not use version_tuple placeholder in setuptools_scm template, #7024\n- get_chunker: fix missing sparse=False argument, #7056\n\nNew features:\n\n- rcompress: do a repo-wide (re)compression, #7037\n- implement pattern support for --match-archives, #6504\n- BORG_LOCK_WAIT=n env var to set default for --lock-wait option, #5279\n\nOther:\n\n- repository.scan: misc. fixes / improvements\n- metadata: differentiate between empty/zero and unknown, #6908\n- CI: test pyfuse3 with python 3.11\n- use more relative imports\n- make borg.testsuite.archiver a package, split archiver tests into many modules\n- support reading new, improved hashindex header format, #6960.\n added version number and num_empty to the HashHeader, fixed alignment.\n- vagrant: upgrade pyinstaller 4.10 -> 5.4.1, use python 3.9.14 for binary build\n- item.pyx: use more Cython (faster, uses less memory), #5763\n\n\nVersion 2.0.0b2 (2022-09-10)\n----------------------------\n\nBug fixes:\n\n- xattrs / extended stat: improve exception handling, #6988\n- fix and refactor replace_placeholders, #6966\n\nNew features:\n\n- support archive timestamps with utc offsets, adapt them when using\n borg transfer to transfer from borg 1.x repos (append +00:00 for UTC).\n- create/recreate/import-tar --timestamp: accept giving timezone via\n its utc offset. defaults to local timezone, if no utc offset is given.\n\nOther changes:\n\n- chunks: have separate encrypted metadata (ctype, clevel, csize, size)\n\n chunk = enc_meta_len16 + encrypted(msgpacked(meta)) + encrypted(compressed(data)).\n\n this breaks repo format compatibility, you need to create fresh repos!\n- repository api: flags support, #6982\n- OpenBSD only - statically link OpenSSL, #6474.\n Avoid conflicting with shared libcrypto from the base OS pulled in via dependencies.\n- restructured source code\n- update diagrams to odg format, #6928\n\nVersion 2.0.0b1 (2022-08-08)\n----------------------------\n\nNew features:\n\n- massively increase archive metadata stream size limit, #1473.\n currently rather testing the code, scalability will improve later, see #6945.\n- rcreate --copy-crypt-key: copy crypt_key from key of other repo, #6710.\n default: create new, random authenticated encryption key.\n- prune/delete --checkpoint-interval=1800 and ctrl-c/SIGINT support, #6284\n\nFixes:\n\n- ctrl-c must not kill important subprocesses, #6912\n- transfer: check whether ID hash method and chunker secret are same.\n add PlaintextKey and AuthenticatedKey support to uses_same_id_hash function.\n- check: try harder to create the key, #5719\n- SaveFile: use a custom mkstemp with mode support, #6933, #6400\n- make setuptools happy, #6874\n- fix misc. compiler warnings\n- list: fix {flags:} formatting, #6081\n\nOther changes:\n\n- new crypto does not need to call ._assert_id(), update code and docs.\n https://github.com/borgbackup/borg/pull/6463#discussion_r925436156\n- check: --verify-data does not need to decompress with new crypto modes\n- Key: crypt_key instead of enc_key + enc_hmac_key, #6611\n- misc. docs updates and improvements\n- CI: test on macOS 12 without fuse / fuse tests\n- repository: add debug logging for issue #6687\n- _version.py: remove trailing blank, add LF at EOF (make pep8 checker happy)\n\n\nVersion 2.0.0a4 (2022-07-17)\n----------------------------\n\nNew features:\n\n- recreate: consider level for recompression, #6698, #3622\n\nOther changes:\n\n- stop using libdeflate\n- CI: add mypy (if we add type hints, it can do type checking)\n- big changes to the source code:\n\n - split up archiver module, transform it into a package\n - use Black for automated code formatting\n - remove some legacy code\n - adapt/fix code for mypy\n- use language_level = 3str for cython (this will be the default in cython 3)\n- docs: document HardLinkManager and hlid, #2388\n\n\nVersion 2.0.0a3 (2022-07-04)\n----------------------------\n\nFixes:\n\n- check repo version, accept old repos only for --other-repo (e.g. rcreate/transfer).\n v2 is the default repo version for borg 2.0. v1 repos must only be used in a\n read-only way, e.g. for --other-repo=V1_REPO with borg init and borg transfer!\n\nNew features:\n\n- transfer: --upgrader=NoOp is the default.\n This is to support general-purpose transfer of archives between related borg2\n repos.\n- transfer: --upgrader=From12To20 must be used to transfer (and convert) archives\n from borg 1.2 repos to borg 2.0 repos.\n\nOther changes:\n\n- removed some deprecated options\n- removed -P (aka --prefix) option, #6806. The option -a (aka --glob-archives)\n can be used for same purpose and is more powerful, e.g.: -a 'PREFIX*'\n- rcreate: always use argon2 kdf for new repos, #6820\n- rcreate: remove legacy encryption modes for new repos, #6490\n\n\nVersion 2.0.0a2 (2022-06-26)\n----------------------------\n\nChanges:\n\n- split repo and archive name into separate args, #948\n\n - use -r or --repo or BORG_REPO env var to give the repository\n - use --other-repo or BORG_OTHER_REPO to give another repo (e.g. borg transfer)\n - use positional argument for archive name or `-a ARCH_GLOB`\n- remove support for scp-style repo specification, use ssh://...\n- simplify stats output: repo ops -> repo stats, archive ops -> archive stats\n- repository index: add payload size (==csize) and flags to NSIndex entries\n- repository index: set/query flags, iteration over flagged items (NSIndex)\n- repository: sync write file in get_fd\n- stats: deduplicated size now, was deduplicated compressed size in borg 1.x\n- remove csize support at most places in the code (chunks index, stats, get_size,\n Item.chunks)\n- replace problematic/ugly hardlink_master approach of borg 1.x by:\n\n - symmetric hlid (all hardlinks pointing to same inode have same hlid)\n - all archived hardlinked regular files have a chunks list\n- borg rcreate --other-repo=OTHER_REPO: reuse key material from OTHER_REPO, #6554.\n This is useful if you want to use borg transfer to transfer archives from an\n existing borg 1.1/1.2 repo. If the chunker secret and the id key and algorithm\n stay the same, the deduplication will also work between past and future backups.\n- borg transfer:\n\n - efficiently copy archives from a borg 1.1/1.2 repo to a new repo.\n uses deduplication and does not decompress/recompress file content data.\n - does some cleanups / fixes / conversions:\n\n - disallow None value for .user/group/chunks/chunks_healthy\n - cleanup msgpack related str/bytes mess, use new msgpack spec, #968\n - obfuscation: fix byte order for size, #6701\n - compression: use the 2 bytes for type and level, #6698\n - use version 2 for new archives\n - convert timestamps int/bigint -> msgpack.Timestamp, see #2323\n - all hardlinks have chunks, maybe chunks_healthy, hlid\n - remove the zlib type bytes hack\n - make sure items with chunks have precomputed size\n - removes the csize element from the tuples in the Item.chunks list\n - clean item of attic 0.13 'acl' bug remnants\n- crypto: see 1.3.0a1 log entry\n- removed \"borg upgrade\" command (not needed any more)\n- compact: removed --cleanup-commits option\n- docs: fixed quickstart and usage docs with new cli command syntax\n- docs: removed the parts talking about potential AES-CTR mode issues\n (we will not use that any more).\n\n\nVersion 1.3.0a1 (2022-04-15)\n----------------------------\n\nAlthough this was released as 1.3.0a1, it can be also seen as 2.0.0a1 as it was\nlater decided to do breaking changes and thus the major release number had to\nbe increased (thus, there will not be a 1.3.0 release, but 2.0.0).\n\nNew features:\n\n- init: new --encryption=(repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305)\n\n - New, better, faster crypto (see encryption-aead diagram in the docs), #6463.\n - New AEAD cipher suites: AES-OCB and CHACHA20-POLY1305.\n - Session keys are derived via HKDF from random session id and master key.\n - Nonces/MessageIVs are counters starting from 0 for each session.\n - AAD: chunk id, key type, messageIV, sessionID are now authenticated also.\n - Solves the potential AES-CTR mode counter management issues of the legacy crypto.\n- init: --key-algorithm=argon2 (new default KDF, older pbkdf2 also still available)\n\n borg key change-passphrase / change-location keeps the key algorithm unchanged.\n- key change-algorithm: to upgrade existing keys to argon2 or downgrade to pbkdf2.\n\n We recommend you to upgrade unless you have to keep the key compatible with older versions of borg.\n- key change-location: usable for repokey <-> keyfile location change\n- benchmark cpu: display benchmarks of cpu bound stuff\n- export-tar: new --tar-format=PAX (default: GNU)\n- import-tar/export-tar: can use PAX format for ctime and atime support\n- import-tar/export-tar: --tar-format=BORG: roundtrip ALL item metadata, #5830\n- repository: create and use version 2 repos only for now\n- repository: implement PUT2: header crc32, overall xxh64, #1704\n\nOther changes:\n\n- require python >= 3.9, #6315\n- simplify libs setup, #6482\n- unbundle most bundled 3rd party code, use libs, #6316\n- use libdeflate.crc32 (Linux and all others) or zlib.crc32 (macOS)\n- repository: code cleanups / simplifications\n- internal crypto api: speedups / cleanups / refactorings / modernisation\n- remove \"borg upgrade\" support for \"attic backup\" repos\n- remove PassphraseKey code and borg key migrate-to-repokey command\n- OpenBSD: build borg with OpenSSL (not: LibreSSL), #6474\n- remove support for LibreSSL, #6474\n- remove support for OpenSSL < 1.1.1\n" }, { "path": "docs/changes_0.x.rst", "content": ".. _changelog_0x:\n\nChange Log 0.x\n==============\n\nVersion 0.30.0 (2016-01-23)\n---------------------------\n\nCompatibility notes:\n\n- The new default logging level is WARNING. Previously, it was INFO, which was\n more verbose. Use -v (or --info) to show once again log level INFO messages.\n See the \"general\" section in the usage docs.\n- For borg create, you need --list (in addition to -v) to see the long file\n list (was needed so you can have e.g. --stats alone without the long list)\n- See below about BORG_DELETE_I_KNOW_WHAT_I_AM_DOING (was:\n BORG_CHECK_I_KNOW_WHAT_I_AM_DOING)\n\nBug fixes:\n\n- fix crash when using borg create --dry-run --keep-tag-files, #570\n- make sure teardown with cleanup happens for Cache and RepositoryCache,\n avoiding leftover locks and TEMP dir contents, #285 (partially), #548\n- fix locking KeyError, partial fix for #502\n- log stats consistently, #526\n- add abbreviated weekday to timestamp format, fixes #496\n- strip whitespace when loading exclusions from file\n- unset LD_LIBRARY_PATH before invoking ssh, fixes strange OpenSSL library\n version warning when using the borg binary, #514\n- add some error handling/fallback for C library loading, #494\n- added BORG_DELETE_I_KNOW_WHAT_I_AM_DOING for check in \"borg delete\", #503\n- remove unused \"repair\" rpc method name\n\nNew features:\n\n- borg create: implement exclusions using regular expression patterns.\n- borg create: implement inclusions using patterns.\n- borg extract: support patterns, #361\n- support different styles for patterns:\n\n - fnmatch (`fm:` prefix, default when omitted), like borg <= 0.29.\n - shell (`sh:` prefix) with `*` not matching directory separators and\n `**/` matching 0..n directories\n - path prefix (`pp:` prefix, for unifying borg create pp1 pp2 into the\n patterns system), semantics like in borg <= 0.29\n - regular expression (`re:`), new!\n- --progress option for borg upgrade (#291) and borg delete \n- update progress indication more often (e.g. for borg create within big\n files or for borg check repo), #500\n- finer chunker granularity for items metadata stream, #547, #487\n- borg create --list is now used (in addition to -v) to enable the verbose\n file list output\n- display borg version below tracebacks, #532\n\nOther changes:\n\n- hashtable size (and thus: RAM and disk consumption) follows a growth policy:\n grows fast while small, grows slower when getting bigger, #527\n- Vagrantfile: use pyinstaller 3.1 to build binaries, freebsd sqlite3 fix,\n fixes #569\n- no separate binaries for centos6 any more because the generic linux binaries\n also work on centos6 (or in general: on systems with a slightly older glibc\n than debian7\n- dev environment: require virtualenv<14.0 so we get a py32 compatible pip\n- docs:\n\n - add space-saving chunks.archive.d trick to FAQ\n - important: clarify -v and log levels in usage -> general, please read!\n - sphinx configuration: create a simple man page from usage docs\n - add a repo server setup example\n - disable unneeded SSH features in authorized_keys examples for security.\n - borg prune only knows \"--keep-within\" and not \"--within\"\n - add gource video to resources docs, #507\n - add netbsd install instructions\n - authors: make it more clear what refers to borg and what to attic\n - document standalone binary requirements, #499\n - rephrase the mailing list section\n - development docs: run build_api and build_usage before tagging release\n - internals docs: hash table max. load factor is 0.75 now\n - markup, typo, grammar, phrasing, clarifications and other fixes.\n - add gcc gcc-c++ to redhat/fedora/corora install docs, fixes #583\n\n\nVersion 0.29.0 (2015-12-13)\n---------------------------\n\nCompatibility notes:\n\n- When upgrading to 0.29.0, you need to upgrade client as well as server\n installations due to the locking and command-line interface changes; otherwise\n you'll get an error message about an RPC protocol mismatch or a wrong command-line\n option.\n If you run a server that needs to support both old and new clients, it is\n suggested that you have a \"borg-0.28.2\" and a \"borg-0.29.0\" command.\n clients then can choose via e.g. \"borg --remote-path=borg-0.29.0 ...\".\n- The default waiting time for a lock changed from infinity to 1 second for a\n better interactive user experience. If the repo you want to access is\n currently locked, borg will now terminate after 1s with an error message.\n If you have scripts that should wait for the lock for a longer time, use\n --lock-wait N (with N being the maximum wait time in seconds).\n\nBug fixes:\n\n- hash table tuning (better chosen hashtable load factor 0.75 and prime initial\n size of 1031 gave ~1000x speedup in some scenarios)\n- avoid creation of an orphan lock for one case, #285\n- --keep-tag-files: fix file mode and multiple tag files in one directory, #432\n- fixes for \"borg upgrade\" (attic repo converter), #466\n- remove --progress isatty magic (and also --no-progress option) again, #476\n- borg init: display proper repo URL\n- fix format of umask in help pages, #463\n\nNew features:\n\n- implement --lock-wait, support timeout for UpgradableLock, #210\n- implement borg break-lock command, #157\n- include system info below traceback, #324\n- sane remote logging, remote stderr, #461:\n\n - remote log output: intercept it and log it via local logging system,\n with \"Remote: \" prefixed to message. log remote tracebacks.\n - remote stderr: output it to local stderr with \"Remote: \" prefixed.\n- add --debug and --info (same as --verbose) to set the log level of the\n builtin logging configuration (which otherwise defaults to warning), #426\n note: there are few messages emitted at DEBUG level currently.\n- optionally configure logging via env var BORG_LOGGING_CONF\n- add --filter option for status characters: e.g. to show only the added\n or modified files (and also errors), use \"borg create -v --filter=AME ...\".\n- more progress indicators, #394\n- use ISO-8601 date and time format, #375\n- \"borg check --prefix\" to restrict archive checking to that name prefix, #206\n\nOther changes:\n\n- hashindex_add C implementation (speed up cache re-sync for new archives)\n- increase FUSE read_size to 1024 (speed up metadata operations)\n- check/delete/prune --save-space: free unused segments quickly, #239\n- increase rpc protocol version to 2 (see also Compatibility notes), #458\n- silence borg by default (via default log level WARNING)\n- get rid of C compiler warnings, #391\n- upgrade OS X FUSE to 3.0.9 on the OS X binary build system\n- use python 3.5.1 to build binaries\n- docs:\n\n - new mailing list borgbackup@python.org, #468\n - readthedocs: color and logo improvements\n - load coverage icons over SSL (avoids mixed content)\n - more precise binary installation steps\n - update release procedure docs about OS X FUSE\n - FAQ entry about unexpected 'A' status for unchanged file(s), #403\n - add docs about 'E' file status\n - add \"borg upgrade\" docs, #464\n - add developer docs about output and logging\n - clarify encryption, add note about client-side encryption\n - add resources section, with videos, talks, presentations, #149\n - Borg moved to Arch Linux [community]\n - fix wrong installation instructions for archlinux\n\n\nVersion 0.28.2 (2015-11-15)\n---------------------------\n\nNew features:\n\n- borg create --exclude-if-present TAGFILE - exclude directories that have the\n given file from the backup. You can additionally give --keep-tag-files to\n preserve just the directory roots and the tag-files (but not back up other\n directory contents), #395, attic #128, attic #142\n\nOther changes:\n\n- do not create docs sources at build time (just have them in the repo),\n completely remove have_cython() hack, do not use the \"mock\" library at build\n time, #384\n- avoid hidden import, make it easier for PyInstaller, easier fix for #218\n- docs:\n\n - add description of item flags / status output, fixes #402\n - explain how to regenerate usage and API files (build_api or\n build_usage) and when to commit usage files directly into git, #384\n - minor install docs improvements\n\n\nVersion 0.28.1 (2015-11-08)\n---------------------------\n\nBug fixes:\n\n- do not try to build api / usage docs for production install,\n fixes unexpected \"mock\" build dependency, #384\n\nOther changes:\n\n- avoid using msgpack.packb at import time\n- fix formatting issue in changes.rst\n- fix build on readthedocs\n\n\nVersion 0.28.0 (2015-11-08)\n---------------------------\n\nCompatibility notes:\n\n- changed return codes (exit codes), see docs. in short:\n old: 0 = ok, 1 = error. now: 0 = ok, 1 = warning, 2 = error\n\nNew features:\n\n- refactor return codes (exit codes), fixes #61\n- add --show-rc option enable \"terminating with X status, rc N\" output, fixes 58, #351\n- borg create backups atime and ctime additionally to mtime, fixes #317\n - extract: support atime additionally to mtime\n - FUSE: support ctime and atime additionally to mtime\n- support borg --version\n- emit a warning if we have a slow msgpack installed\n- borg list --prefix=thishostname- REPO, fixes #205\n- Debug commands (do not use except if you know what you do: debug-get-obj,\n debug-put-obj, debug-delete-obj, debug-dump-archive-items.\n\nBug fixes:\n\n- setup.py: fix bug related to BORG_LZ4_PREFIX processing\n- fix \"check\" for repos that have incomplete chunks, fixes #364\n- borg mount: fix unlocking of repository at umount time, fixes #331\n- fix reading files without touching their atime, #334\n- non-ascii ACL fixes for Linux, FreeBSD and OS X, #277\n- fix acl_use_local_uid_gid() and add a test for it, attic #359\n- borg upgrade: do not upgrade repositories in place by default, #299\n- fix cascading failure with the index conversion code, #269\n- borg check: implement 'cmdline' archive metadata value decoding, #311\n- fix RobustUnpacker, it missed some metadata keys (new atime and ctime keys\n were missing, but also bsdflags). add check for unknown metadata keys.\n- create from stdin: also save atime, ctime (cosmetic)\n- use default_notty=False for confirmations, fixes #345\n- vagrant: fix msgpack installation on centos, fixes #342\n- deal with unicode errors for symlinks in same way as for regular files and\n have a helpful warning message about how to fix wrong locale setup, fixes #382\n- add ACL keys the RobustUnpacker must know about\n\nOther changes:\n\n- improve file size displays, more flexible size formatters\n- explicitly commit to the units standard, #289\n- archiver: add E status (means that an error occurred when processing this\n (single) item\n- do binary releases via \"github releases\", closes #214\n- create: use -x and --one-file-system (was: --do-not-cross-mountpoints), #296\n- a lot of changes related to using \"logging\" module and screen output, #233\n- show progress display if on a tty, output more progress information, #303\n- factor out status output so it is consistent, fix surrogates removal,\n maybe fixes #309\n- move away from RawConfigParser to ConfigParser\n- archive checker: better error logging, give chunk_id and sequence numbers\n (can be used together with borg debug-dump-archive-items).\n- do not mention the deprecated passphrase mode\n- emit a deprecation warning for --compression N (giving a just a number)\n- misc .coverragerc fixes (and coverage measurement improvements), fixes #319\n- refactor confirmation code, reduce code duplication, add tests\n- prettier error messages, fixes #307, #57\n- tests:\n\n - add a test to find disk-full issues, #327\n - travis: also run tests on Python 3.5\n - travis: use tox -r so it rebuilds the tox environments\n - test the generated pyinstaller-based binary by archiver unit tests, #215\n - vagrant: tests: announce whether fakeroot is used or not\n - vagrant: add vagrant user to fuse group for debianoid systems also\n - vagrant: llfuse install on darwin needs pkgconfig installed\n - vagrant: use pyinstaller from develop branch, fixes #336\n - benchmarks: test create, extract, list, delete, info, check, help, fixes #146\n - benchmarks: test with both the binary and the python code\n - archiver tests: test with both the binary and the python code, fixes #215\n - make basic test more robust\n- docs:\n\n - moved docs to borgbackup.readthedocs.org, #155\n - a lot of fixes and improvements, use mobile-friendly RTD standard theme\n - use zlib,6 compression in some examples, fixes #275\n - add missing rename usage to docs, closes #279\n - include the help offered by borg help in the usage docs, fixes #293\n - include a list of major changes compared to attic into README, fixes #224\n - add OS X install instructions, #197\n - more details about the release process, #260\n - fix linux glibc requirement (binaries built on debian7 now)\n - build: move usage and API generation to setup.py\n - update docs about return codes, #61\n - remove api docs (too much breakage on rtd)\n - borgbackup install + basics presentation (asciinema)\n - describe the current style guide in documentation\n - add section about debug commands\n - warn about not running out of space\n - add example for rename\n - improve chunker params docs, fixes #362\n - minor development docs update\n\n\nVersion 0.27.0 (2015-10-07)\n---------------------------\n\nNew features:\n\n- \"borg upgrade\" command - attic -> borg one time converter / migration, #21\n- temporary hack to avoid using lots of disk space for chunks.archive.d, #235:\n To use it: rm -rf chunks.archive.d ; touch chunks.archive.d\n- respect XDG_CACHE_HOME, attic #181\n- add support for arbitrary SSH commands, attic #99\n- borg delete --cache-only REPO (only delete cache, not REPO), attic #123\n\n\nBug fixes:\n\n- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow\n down observed when running the Centos6-built binary on Ubuntu, #222\n- do not crash on empty lock.roster, fixes #232\n- fix multiple issues with the cache config version check, #234\n- fix segment entry header size check, attic #352\n plus other error handling improvements / code deduplication there.\n- always give segment and offset in repo IntegrityErrors\n\n\nOther changes:\n\n- stop producing binary wheels, remove docs about it, #147\n- docs:\n - add warning about prune\n - generate usage include files only as needed\n - development docs: add Vagrant section\n - update / improve / reformat FAQ\n - hint to single-file pyinstaller binaries from README\n\n\nVersion 0.26.1 (2015-09-28)\n---------------------------\n\nThis is a minor update, just docs and new pyinstaller binaries.\n\n- docs update about python and binary requirements\n- better docs for --read-special, fix #220\n- re-built the binaries, fix #218 and #213 (glibc version issue)\n- update web site about single-file pyinstaller binaries\n\nNote: if you did a python-based installation, there is no need to upgrade.\n\n\nVersion 0.26.0 (2015-09-19)\n---------------------------\n\nNew features:\n\n- Faster cache sync (do all in one pass, remove tar/compression stuff), #163\n- BORG_REPO env var to specify the default repo, #168\n- read special files as if they were regular files, #79\n- implement borg create --dry-run, attic issue #267\n- Normalize paths before pattern matching on OS X, #143\n- support OpenBSD and NetBSD (except xattrs/ACLs)\n- support / run tests on Python 3.5\n\nBug fixes:\n\n- borg mount repo: use absolute path, attic #200, attic #137\n- chunker: use off_t to get 64bit on 32bit platform, #178\n- initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0)\n- fix reaction to \"no\" answer at delete repo prompt, #182\n- setup.py: detect lz4.h header file location\n- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194)\n- fix for obtaining ``char *`` from temporary Python value (old code causes\n a compile error on Mint 17.2)\n- llfuse 0.41 install troubles on some platforms, require < 0.41\n (UnicodeDecodeError exception due to non-ascii llfuse setup.py)\n- cython code: add some int types to get rid of unspecific python add /\n subtract operations (avoid ``undefined symbol FPE_``... error on some platforms)\n- fix verbose mode display of stdin backup\n- extract: warn if a include pattern never matched, fixes #209,\n implement counters for Include/ExcludePatterns\n- archive names with slashes are invalid, attic issue #180\n- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined -\n fixes building on OpenBSD.\n\nOther changes:\n\n- detect inconsistency / corruption / hash collision, #170\n- replace versioneer with setuptools_scm, #106\n- docs:\n\n - pkg-config is needed for llfuse installation\n - be more clear about pruning, attic issue #132\n- unit tests:\n\n - xattr: ignore security.selinux attribute showing up\n - ext3 seems to need a bit more space for a sparse file\n - do not test lzma level 9 compression (avoid MemoryError)\n - work around strange mtime granularity issue on netbsd, fixes #204\n - ignore st_rdev if file is not a block/char device, fixes #203\n - stay away from the setgid and sticky mode bits\n- use Vagrant to do easy cross-platform testing (#196), currently:\n\n - Debian 7 \"wheezy\" 32bit, Debian 8 \"jessie\" 64bit\n - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit\n - Centos 7 64bit\n - FreeBSD 10.2 64bit\n - OpenBSD 5.7 64bit\n - NetBSD 6.1.5 64bit\n - Darwin (OS X Yosemite)\n\n\nVersion 0.25.0 (2015-08-29)\n---------------------------\n\nCompatibility notes:\n\n- lz4 compression library (liblz4) is a new requirement (#156)\n- the new compression code is very compatible: as long as you stay with zlib\n compression, older borg releases will still be able to read data from a\n repo/archive made with the new code (note: this is not the case for the\n default \"none\" compression, use \"zlib,0\" if you want a \"no compression\" mode\n that can be read by older borg). Also the new code is able to read repos and\n archives made with older borg versions (for all zlib levels 0..9).\n\nDeprecations:\n\n- --compression N (with N being a number, as in 0.24) is deprecated.\n We keep the --compression 0..9 for now not to break scripts, but it is\n deprecated and will be removed later, so better fix your scripts now:\n --compression 0 (as in 0.24) is the same as --compression zlib,0 (now).\n BUT: if you do not want compression, use --compression none\n (which is the default).\n --compression 1 (in 0.24) is the same as --compression zlib,1 (now)\n --compression 9 (in 0.24) is the same as --compression zlib,9 (now)\n\nNew features:\n\n- create --compression none (default, means: do not compress, just pass through\n data \"as is\". this is more efficient than zlib level 0 as used in borg 0.24)\n- create --compression lz4 (super-fast, but not very high compression)\n- create --compression zlib,N (slower, higher compression, default for N is 6)\n- create --compression lzma,N (slowest, highest compression, default N is 6)\n- honor the nodump flag (UF_NODUMP) and do not back up such items\n- list --short just outputs a simple list of the files/directories in an archive\n\nBug fixes:\n\n- fixed --chunker-params parameter order confusion / malfunction, fixes #154\n- close fds of segments we delete (during compaction)\n- close files which fell out the lrucache\n- fadvise DONTNEED now is only called for the byte range actually read, not for\n the whole file, fixes #158.\n- fix issue with negative \"all archives\" size, fixes #165\n- restore_xattrs: ignore if setxattr fails with EACCES, fixes #162\n\nOther changes:\n\n- remove fakeroot requirement for tests, tests run faster without fakeroot\n (test setup does not fail any more without fakeroot, so you can run with or\n without fakeroot), fixes #151 and #91.\n- more tests for archiver\n- recover_segment(): don't assume we have an fd for segment\n- lrucache refactoring / cleanup, add dispose function, py.test tests\n- generalize hashindex code for any key length (less hardcoding)\n- lock roster: catch file not found in remove() method and ignore it\n- travis CI: use requirements file\n- improved docs:\n\n - replace hack for llfuse with proper solution (install libfuse-dev)\n - update docs about compression\n - update development docs about fakeroot\n - internals: add some words about lock files / locking system\n - support: mention BountySource and for what it can be used\n - theme: use a lighter green\n - add pypi, wheel, dist package based install docs\n - split install docs into system-specific preparations and generic instructions\n\n\nVersion 0.24.0 (2015-08-09)\n---------------------------\n\nIncompatible changes (compared to 0.23):\n\n- borg now always issues --umask NNN option when invoking another borg via ssh\n on the repository server. By that, it's making sure it uses the same umask\n for remote repos as for local ones. Because of this, you must upgrade both\n server and client(s) to 0.24.\n- the default umask is 077 now (if you do not specify via --umask) which might\n be a different one as you used previously. The default umask avoids that\n you accidentally give access permissions for group and/or others to files\n created by borg (e.g. the repository).\n\nDeprecations:\n\n- \"--encryption passphrase\" mode is deprecated, see #85 and #97.\n See the new \"--encryption repokey\" mode for a replacement.\n\nNew features:\n\n- borg create --chunker-params ... to configure the chunker, fixes #16\n (attic #302, attic #300, and somehow also #41).\n This can be used to reduce memory usage caused by chunk management overhead,\n so borg does not create a huge chunks index/repo index and eats all your RAM\n if you back up lots of data in huge files (like VM disk images).\n See docs/misc/create_chunker-params.txt for more information.\n- borg info now reports chunk counts in the chunk index.\n- borg create --compression 0..9 to select zlib compression level, fixes #66\n (attic #295).\n- borg init --encryption repokey (to store the encryption key into the repo),\n fixes #85\n- improve at-end error logging, always log exceptions and set exit_code=1\n- LoggedIO: better error checks / exceptions / exception handling\n- implement --remote-path to allow non-default-path borg locations, #125\n- implement --umask M and use 077 as default umask for better security, #117\n- borg check: give a named single archive to it, fixes #139\n- cache sync: show progress indication\n- cache sync: reimplement the chunk index merging in C\n\nBug fixes:\n\n- fix segfault that happened for unreadable files (chunker: n needs to be a\n signed size_t), #116\n- fix the repair mode, #144\n- repo delete: add destroy to allowed rpc methods, fixes issue #114\n- more compatible repository locking code (based on mkdir), maybe fixes #92\n (attic #317, attic #201).\n- better Exception msg if no Borg is installed on the remote repo server, #56\n- create a RepositoryCache implementation that can cope with >2GiB,\n fixes attic #326.\n- fix Traceback when running check --repair, attic #232\n- clarify help text, fixes #73.\n- add help string for --no-files-cache, fixes #140\n\nOther changes:\n\n- improved docs:\n\n - added docs/misc directory for misc. writeups that won't be included\n \"as is\" into the html docs.\n - document environment variables and return codes (attic #324, attic #52)\n - web site: add related projects, fix web site url, IRC #borgbackup\n - Fedora/Fedora-based install instructions added to docs\n - Cygwin-based install instructions added to docs\n - updated AUTHORS\n - add FAQ entries about redundancy / integrity\n - clarify that borg extract uses the cwd as extraction target\n - update internals doc about chunker params, memory usage and compression\n - added docs about development\n - add some words about resource usage in general\n - document how to back up a raw disk\n - add note about how to run borg from virtual env\n - add solutions for (ll)fuse installation problems\n - document what borg check does, fixes #138\n - reorganize borgbackup.github.io sidebar, prev/next at top\n - deduplicate and refactor the docs / README.rst\n\n- use borg-tmp as prefix for temporary files / directories\n- short prune options without \"keep-\" are deprecated, do not suggest them\n- improved tox configuration\n- remove usage of unittest.mock, always use mock from pypi\n- use entrypoints instead of scripts, for better use of the wheel format and\n modern installs\n- add requirements.d/development.txt and modify tox.ini\n- use travis-ci for testing based on Linux and (new) OS X\n- use coverage.py, pytest-cov and codecov.io for test coverage support\n\nI forgot to list some stuff already implemented in 0.23.0, here they are:\n\nNew features:\n\n- efficient archive list from manifest, meaning a big speedup for slow\n repo connections and \"list \", \"delete \", \"prune\" (attic #242,\n attic #167)\n- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18\n- hashindex: improve error messages\n\nOther changes:\n\n- explicitly specify binary mode to open binary files\n- some easy micro optimizations\n\n\nVersion 0.23.0 (2015-06-11)\n---------------------------\n\nIncompatible changes (compared to attic, fork related):\n\n- changed sw name and cli command to \"borg\", updated docs\n- package name (and name in urls) uses \"borgbackup\" to have fewer collisions\n- changed repo / cache internal magic strings from ATTIC* to BORG*,\n changed cache location to .cache/borg/ - this means that it currently won't\n accept attic repos (see issue #21 about improving that)\n\nBug fixes:\n\n- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185\n- fix traceback when trying to do unsupported passphrase change, fixes attic #189\n- datetime does not like the year 10.000, fixes attic #139\n- fix \"info\" all archives stats, fixes attic #183\n- fix parsing with missing microseconds, fixes attic #282\n- fix misleading hint the fuse ImportError handler gave, fixes attic #237\n- check unpacked data from RPC for tuple type and correct length, fixes attic #127\n- fix Repository._active_txn state when lock upgrade fails\n- give specific path to xattr.is_enabled(), disable symlink setattr call that\n always fails\n- fix test setup for 32bit platforms, partial fix for attic #196\n- upgraded versioneer, PEP440 compliance, fixes attic #257\n\nNew features:\n\n- less memory usage: add global option --no-cache-files\n- check --last N (only check the last N archives)\n- check: sort archives in reverse time order\n- rename repo::oldname newname (rename repository)\n- create -v output more informative\n- create --progress (backup progress indicator)\n- create --timestamp (utc string or reference file/dir)\n- create: if \"-\" is given as path, read binary from stdin\n- extract: if --stdout is given, write all extracted binary data to stdout\n- extract --sparse (simple sparse file support)\n- extra debug information for 'fread failed'\n- delete (deletes whole repo + local cache)\n- FUSE: reflect deduplication in allocated blocks\n- only allow whitelisted RPC calls in server mode\n- normalize source/exclude paths before matching\n- use posix_fadvise not to spoil the OS cache, fixes attic #252\n- toplevel error handler: show tracebacks for better error analysis\n- sigusr1 / sigint handler to print current file infos - attic PR #286\n- RPCError: include the exception args we get from remote\n\nOther changes:\n\n- source: misc. cleanups, pep8, style\n- docs and faq improvements, fixes, updates\n- cleanup crypto.pyx, make it easier to adapt to other AES modes\n- do os.fsync like recommended in the python docs\n- source: Let chunker optionally work with os-level file descriptor.\n- source: Linux: remove duplicate os.fsencode calls\n- source: refactor _open_rb code a bit, so it is more consistent / regular\n- source: refactor indicator (status) and item processing\n- source: use py.test for better testing, flake8 for code style checks\n- source: fix tox >=2.0 compatibility (test runner)\n- pypi package: add python version classifiers, add FreeBSD to platforms\n\n\nAttic Changelog\n---------------\n\nHere you can see the full list of changes between each Attic release until Borg\nforked from Attic:\n\nVersion 0.17\n~~~~~~~~~~~~\n\n(bugfix release, released on X)\n\n- Fix hashindex ARM memory alignment issue (#309)\n- Improve hashindex error messages (#298)\n\nVersion 0.16\n~~~~~~~~~~~~\n\n(bugfix release, released on May 16, 2015)\n\n- Fix typo preventing the security confirmation prompt from working (#303)\n- Improve handling of systems with improperly configured file system encoding (#289)\n- Fix \"All archives\" output for attic info. (#183)\n- More user friendly error message when repository key file is not found (#236)\n- Fix parsing of iso 8601 timestamps with zero microseconds (#282)\n\nVersion 0.15\n~~~~~~~~~~~~\n\n(bugfix release, released on Apr 15, 2015)\n\n- xattr: Be less strict about unknown/unsupported platforms (#239)\n- Reduce repository listing memory usage (#163).\n- Fix BrokenPipeError for remote repositories (#233)\n- Fix incorrect behavior with two character directory names (#265, #268)\n- Require approval before accessing relocated/moved repository (#271)\n- Require approval before accessing previously unknown unencrypted repositories (#271)\n- Fix issue with hash index files larger than 2GB.\n- Fix Python 3.2 compatibility issue with noatime open() (#164)\n- Include missing pyx files in dist files (#168)\n\nVersion 0.14\n~~~~~~~~~~~~\n\n(feature release, released on Dec 17, 2014)\n\n- Added support for stripping leading path segments (#95)\n \"attic extract --strip-segments X\"\n- Add workaround for old Linux systems without acl_extended_file_no_follow (#96)\n- Add MacPorts' path to the default openssl search path (#101)\n- HashIndex improvements, eliminates unnecessary IO on low memory systems.\n- Fix \"Number of files\" output for attic info. (#124)\n- limit create file permissions so files aren't read while restoring\n- Fix issue with empty xattr values (#106)\n\nVersion 0.13\n~~~~~~~~~~~~\n\n(feature release, released on Jun 29, 2014)\n\n- Fix sporadic \"Resource temporarily unavailable\" when using remote repositories\n- Reduce file cache memory usage (#90)\n- Faster AES encryption (utilizing AES-NI when available)\n- Experimental Linux, OS X and FreeBSD ACL support (#66)\n- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56)\n- Fix bug where xattrs on symlinks were not correctly restored\n- Added cachedir support. CACHEDIR.TAG compatible cache directories\n can now be excluded using ``--exclude-caches`` (#74)\n- Fix crash on extreme mtime timestamps (year 2400+) (#81)\n- Fix Python 3.2 specific lockf issue (EDEADLK)\n\nVersion 0.12\n~~~~~~~~~~~~\n\n(feature release, released on April 7, 2014)\n\n- Python 3.4 support (#62)\n- Various documentation improvements a new style\n- ``attic mount`` now supports mounting an entire repository not only\n individual archives (#59)\n- Added option to restrict remote repository access to specific path(s):\n ``attic serve --restrict-to-path X`` (#51)\n- Include \"all archives\" size information in \"--stats\" output. (#54)\n- Added ``--stats`` option to ``attic delete`` and ``attic prune``\n- Fixed bug where ``attic prune`` used UTC instead of the local time zone\n when determining which archives to keep.\n- Switch to SI units (Power of 1000 instead 1024) when printing file sizes\n\nVersion 0.11\n~~~~~~~~~~~~\n\n(feature release, released on March 7, 2014)\n\n- New \"check\" command for repository consistency checking (#24)\n- Documentation improvements\n- Fix exception during \"attic create\" with repeated files (#39)\n- New \"--exclude-from\" option for attic create/extract/verify.\n- Improved archive metadata deduplication.\n- \"attic verify\" has been deprecated. Use \"attic extract --dry-run\" instead.\n- \"attic prune --hourly|daily|...\" has been deprecated.\n Use \"attic prune --keep-hourly|daily|...\" instead.\n- Ignore xattr errors during \"extract\" if not supported by the filesystem. (#46)\n\nVersion 0.10\n~~~~~~~~~~~~\n\n(bugfix release, released on Jan 30, 2014)\n\n- Fix deadlock when extracting 0 sized files from remote repositories\n- \"--exclude\" wildcard patterns are now properly applied to the full path\n not just the file name part (#5).\n- Make source code endianness agnostic (#1)\n\nVersion 0.9\n~~~~~~~~~~~\n\n(feature release, released on Jan 23, 2014)\n\n- Remote repository speed and reliability improvements.\n- Fix sorting of segment names to ignore NFS left over files. (#17)\n- Fix incorrect display of time (#13)\n- Improved error handling / reporting. (#12)\n- Use fcntl() instead of flock() when locking repository/cache. (#15)\n- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9)\n- Improved libcrypto path detection (#23).\n\nVersion 0.8.1\n~~~~~~~~~~~~~\n\n(bugfix release, released on Oct 4, 2013)\n\n- Fix segmentation fault issue.\n\nVersion 0.8\n~~~~~~~~~~~\n\n(feature release, released on Oct 3, 2013)\n\n- Fix xattr issue when backing up sshfs filesystems (#4)\n- Fix issue with excessive index file size (#6)\n- Support access of read only repositories.\n- New syntax to enable repository encryption:\n attic init --encryption=\"none|passphrase|keyfile\".\n- Detect and abort if repository is older than the cache.\n\n\nVersion 0.7\n~~~~~~~~~~~\n\n(feature release, released on Aug 5, 2013)\n\n- Ported to FreeBSD\n- Improved documentation\n- Experimental: Archives mountable as FUSE filesystems.\n- The \"user.\" prefix is no longer stripped from xattrs on Linux\n\n\nVersion 0.6.1\n~~~~~~~~~~~~~\n\n(bugfix release, released on July 19, 2013)\n\n- Fixed an issue where mtime was not always correctly restored.\n\n\nVersion 0.6\n~~~~~~~~~~~\n\nFirst public release on July 9, 2013\n" }, { "path": "docs/changes_1.x.rst", "content": ".. _important_notes_1x:\n\nImportant notes 1.x\n===================\n\nThis section provides information about security and corruption issues.\n\n.. _archives_tam_vuln:\n\nPre-1.2.5 archives spoofing vulnerability (CVE-2023-36811)\n----------------------------------------------------------\n\nA flaw in the cryptographic authentication scheme in Borg allowed an attacker to\nfake archives and potentially indirectly cause backup data loss in the repository.\n\nThe attack requires an attacker to be able to\n\n1. insert files (with no additional headers) into backups\n2. gain write access to the repository\n\nThis vulnerability does not disclose plaintext to the attacker, nor does it\naffect the authenticity of existing archives.\n\nCreating plausible fake archives may be feasible for empty or small archives,\nbut is unlikely for large archives.\n\nThe fix enforces checking the TAM authentication tag of archives at critical\nplaces. Borg now considers archives without TAM as garbage or an attack.\n\nWe are not aware of others having discovered, disclosed or exploited this vulnerability.\n\nBelow, if we speak of borg 1.2.6, we mean a borg version >= 1.2.6 **or** a\nborg version that has the relevant security patches for this vulnerability applied\n(could be also an older version in that case).\n\nSteps you must take to upgrade a repository (this applies to all kinds of repos\nno matter what encryption mode they use, including \"none\"):\n\n1. Upgrade all clients using this repository to borg 1.2.6.\n Note: it is not required to upgrade a server, except if the server-side borg\n is also used as a client (and not just for \"borg serve\").\n\n Do **not** run ``borg check`` with borg 1.2.6 before completing the upgrade steps:\n\n - ``borg check`` would complain about archives without a valid archive TAM.\n - ``borg check --repair`` would remove such archives!\n2. Run: ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg info --debug 2>&1 | grep TAM | grep -i manifest``\n\n a) If you get \"TAM-verified manifest\", continue with 3.\n b) If you get \"Manifest TAM not found and not required\", run\n ``borg upgrade --tam --force `` *on every client*.\n\n3. Run: ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg list --consider-checkpoints --format='{name} {time} tam:{tam}{NL}' ``\n\n \"tam:verified\" means that the archive has a valid TAM authentication.\n \"tam:none\" is expected as output for archives created by borg <1.0.9.\n \"tam:none\" is also expected for archives resulting from a borg rename\n or borg recreate operation (see #7791).\n \"tam:none\" could also come from archives created by an attacker.\n You should verify that \"tam:none\" archives are authentic and not malicious\n (== have good content, have correct timestamp, can be extracted successfully).\n In case you find crappy/malicious archives, you must delete them before proceeding.\n In low-risk, trusted environments, you may decide on your own risk to skip step 3\n and just trust in everything being OK.\n\n4. If there are no tam:none archives left at this point, you can skip this step.\n Run ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg upgrade --archives-tam ``.\n This will unconditionally add a correct archive TAM to all archives not having one.\n ``borg check`` would consider TAM-less or invalid-TAM archives as garbage or a potential attack.\n To see that all archives now are \"tam:verified\" run: ``borg list --consider-checkpoints --format='{name} {time} tam:{tam}{NL}' ``\n\n5. Please note that you should never use BORG_WORKAROUNDS=ignore_invalid_archive_tam\n for normal production operations - it is only needed once to get the archives in a\n repository into a good state. All archives have a valid TAM now.\n\nVulnerability timeline:\n\n* 2023-06-13: Vulnerability discovered during code review by Thomas Waldmann\n* 2023-06-13...: Work on fixing the issue, upgrade procedure, docs.\n* 2023-06-30: CVE was assigned via Github CNA\n* 2023-06-30 .. 2023-08-29: Fixed issue, code review, docs, testing.\n* 2023-08-30: Released fixed version 1.2.5 (broken upgrade procedure for some repos)\n* 2023-08-31: Released fixed version 1.2.6 (fixes upgrade procedure)\n\n.. _hashindex_set_bug:\n\nPre-1.1.11 potential index corruption / data loss issue\n-------------------------------------------------------\n\nA bug was discovered in our hashtable code, see issue #4829.\nThe code is used for the client-side chunks cache and the server-side repo index.\n\nAlthough borg uses the hashtables very heavily, the index corruption did not\nhappen too frequently, because it needed specific conditions to happen.\n\nData loss required even more specific conditions, so it should be rare (and\nalso detectable via borg check).\n\nYou might be affected if borg crashed with / complained about:\n\n- AssertionError: Corrupted segment reference count - corrupted index or hints\n- ObjectNotFound: Object with key ... not found in repository ...\n- Index mismatch for key b'...'. (..., ...) != (-1, -1)\n- ValueError: stats_against: key contained in self but not in master_index.\n\nAdvised procedure to fix any related issue in your indexes/caches:\n\n- install fixed borg code (on client AND server)\n- for all of your clients and repos remove the cache by:\n\n borg delete --cache-only YOURREPO\n\n (later, the cache will be re-built automatically)\n- for all your repos, rebuild the repo index by:\n\n borg check --repair YOURREPO\n\n This will also check all archives and detect if there is any data-loss issue.\n\nAffected branches / releases:\n\n- fd06497 introduced the bug into 1.1-maint branch - it affects all borg 1.1.x since 1.1.0b4.\n- fd06497 introduced the bug into master branch - it affects all borg 1.2.0 alpha releases.\n- c5cd882 introduced the bug into 1.0-maint branch - it affects all borg 1.0.x since 1.0.11rc1.\n\nThe bug was fixed by:\n\n- 701159a fixes the bug in 1.1-maint branch - will be released with borg 1.1.11.\n- fa63150 fixes the bug in master branch - will be released with borg 1.2.0a8.\n- 7bb90b6 fixes the bug in 1.0-maint branch. Branch is EOL, no new release is planned as of now.\n\n.. _broken_validator:\n\nPre-1.1.4 potential data corruption issue\n-----------------------------------------\n\nA data corruption bug was discovered in borg check --repair, see issue #3444.\n\nThis is a 1.1.x regression, releases < 1.1 (e.g. 1.0.x) are not affected.\n\nTo avoid data loss, you must not run borg check --repair using an unfixed version\nof borg 1.1.x. The first official release that has the fix is 1.1.4.\n\nPackage maintainers may have applied the fix to updated packages of 1.1.x (x<4)\nthough, see the package maintainer's package changelog to make sure.\n\nIf you never had missing item metadata chunks, the bug has not affected you\neven if you did run borg check --repair with an unfixed version.\n\nWhen borg check --repair tried to repair corrupt archives that miss item metadata\nchunks, the resync to valid metadata in still present item metadata chunks\nmalfunctioned. This was due to a broken validator that considered all (even valid)\nitem metadata as invalid. As they were considered invalid, borg discarded them.\nPractically, that means the affected files, directories or other fs objects were\ndiscarded from the archive.\n\nDue to the malfunction, the process was extremely slow, but if you let it\ncomplete, borg would have created a \"repaired\" archive that has lost a lot of items.\nIf you interrupted borg check --repair because it was so strangely slow (killing\nborg somehow, e.g. Ctrl-C) the transaction was rolled back and no corruption occurred.\n\nThe log message indicating the precondition for the bug triggering looks like:\n\n item metadata chunk missing [chunk: 001056_bdee87d...a3e50d]\n\nIf you never had that in your borg check --repair runs, you're not affected.\n\nBut if you're unsure or you actually have seen that, better check your archives.\nBy just using \"borg list repo::archive\" you can see if all expected filesystem\nitems are listed.\n\n.. _tam_vuln:\n\nPre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099)\n----------------------------------------------------------\n\nA flaw in the cryptographic authentication scheme in Borg allowed an attacker\nto spoof the manifest. The attack requires an attacker to be able to\n\n1. insert files (with no additional headers) into backups\n2. gain write access to the repository\n\nThis vulnerability does not disclose plaintext to the attacker, nor does it\naffect the authenticity of existing archives.\n\nThe vulnerability allows an attacker to create a spoofed manifest (the list of archives).\nCreating plausible fake archives may be feasible for small archives, but is unlikely\nfor large archives.\n\nThe fix adds a separate authentication tag to the manifest. For compatibility\nwith prior versions this authentication tag is *not* required by default\nfor existing repositories. Repositories created with 1.0.9 and later require it.\n\nSteps you should take:\n\n1. Upgrade all clients to 1.0.9 or later.\n2. Run ``borg upgrade --tam `` *on every client* for *each* repository.\n3. This will list all archives, including archive IDs, for easy comparison with your logs.\n4. Done.\n\nPrior versions can access and modify repositories with this measure enabled, however,\nto 1.0.9 or later their modifications are indiscernible from an attack and will\nraise an error until the below procedure is followed. We are aware that this can\nbe annoying in some circumstances, but don't see a way to fix the vulnerability\notherwise.\n\nIn case a version prior to 1.0.9 is used to modify a repository where above procedure\nwas completed, and now you get an error message from other clients:\n\n1. ``borg upgrade --tam --force `` once with *any* client suffices.\n\nThis attack is mitigated by:\n\n- Noting/logging ``borg list``, ``borg info``, or ``borg create --stats``, which\n contain the archive IDs.\n\nWe are not aware of others having discovered, disclosed or exploited this vulnerability.\n\nVulnerability time line:\n\n* 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore)\n* 2016-11-20: First patch\n* 2016-12-20: Released fixed version 1.0.9\n* 2017-01-02: CVE was assigned\n* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source)\n\n.. _attic013_check_corruption:\n\nPre-1.0.9 potential data loss\n-----------------------------\n\nIf you have archives in your repository that were made with attic <= 0.13\n(and later migrated to borg), running borg check would report errors in these\narchives. See issue #1837.\n\nThe reason for this is a invalid (and useless) metadata key that was\nalways added due to a bug in these old attic versions.\n\nIf you run borg check --repair, things escalate quickly: all archive items\nwith invalid metadata will be killed. Due to that attic bug, that means all\nitems in all archives made with these old attic versions.\n\n\nPre-1.0.4 potential repo corruption\n-----------------------------------\n\nSome external errors (like network or disk I/O errors) could lead to\ncorruption of the backup repository due to issue #1138.\n\nA sign that this happened is if \"E\" status was reported for a file that can\nnot be explained by problems with the source file. If you still have logs from\n\"borg create -v --list\", you can check for \"E\" status.\n\nHere is what could cause corruption and what you can do now:\n\n1) I/O errors (e.g. repo disk errors) while writing data to repo.\n\nThis could lead to corrupted segment files.\n\nFix::\n\n # check for corrupt chunks / segments:\n borg check -v --repository-only REPO\n\n # repair the repo:\n borg check -v --repository-only --repair REPO\n\n # make sure everything is fixed:\n borg check -v --repository-only REPO\n\n2) Unreliable network / unreliable connection to the repo.\n\nThis could lead to archive metadata corruption.\n\nFix::\n\n # check for corrupt archives:\n borg check -v --archives-only REPO\n\n # delete the corrupt archives:\n borg delete --force REPO::CORRUPT_ARCHIVE\n\n # make sure everything is fixed:\n borg check -v --archives-only REPO\n\n3) In case you want to do more intensive checking.\n\nThe best check that everything is ok is to run a dry-run extraction::\n\n borg extract -v --dry-run REPO::ARCHIVE\n\n\n.. _upgradenotes:\n\nUpgrade Notes\n=============\n\nborg 1.1.x to 1.2.x\n-------------------\n\nSome things can be recommended for the upgrade process from borg 1.1.x\n(please also read the important compatibility notes below):\n\n- first upgrade to a recent 1.1.x release - especially if you run some older\n 1.1.* or even 1.0.* borg release.\n- using that, run at least one `borg create` (your normal backup), `prune`\n and especially a `check` to see everything is in a good state.\n- check the output of `borg check` - if there is anything special, consider\n a `borg check --repair` followed by another `borg check`.\n- if everything is fine so far (borg check reports no issues), you can consider\n upgrading to 1.2.x. if not, please first fix any already existing issue.\n- if you want to play safer, first **create a backup of your borg repository**.\n- upgrade to latest borg 1.2.x release (you could use the fat binary from\n github releases page)\n- borg 1.2.6 has a security fix for the pre-1.2.5 archives spoofing vulnerability\n (CVE-2023-36811), see details and necessary upgrade procedure described above.\n- run `borg compact --cleanup-commits` to clean up a ton of 17 bytes long files\n in your repo caused by a borg 1.1 bug\n- run `borg check` again (now with borg 1.2.x) and check if there is anything\n special.\n- run `borg info` (with borg 1.2.x) to build the local pre12-meta cache (can\n take significant time, but after that it will be fast) - for more details\n see below.\n- check the compatibility notes (see below) and adapt your scripts, if needed.\n- if you run into any issues, please check the github issue tracker before\n posting new issues there or elsewhere.\n\nIf you follow this procedure, you can help avoiding that we get a lot of\n\"borg 1.2\" issue reports that are not really 1.2 issues, but existed before\nand maybe just were not noticed.\n\nCompatibility notes:\n\n- matching of path patterns has been aligned with borg storing relative paths.\n Borg archives file paths without leading slashes. Previously, include/exclude\n patterns could contain leading slashes. You should check your patterns and\n remove leading slashes.\n- dropped support / testing for older Pythons, minimum requirement is 3.8.\n In case your OS does not provide Python >= 3.8, consider using our binary,\n which does not need an external Python interpreter. Or continue using\n borg 1.1.x, which is still supported.\n- freeing repository space only happens when \"borg compact\" is invoked.\n- mount: the default for --numeric-ids is False now (same as borg extract)\n- borg create --noatime is deprecated. Not storing atime is the default behaviour\n now (use --atime if you want to store the atime).\n- --prefix is deprecated, use -a / --glob-archives, see #6806\n- list: corrected mix-up of \"isomtime\" and \"mtime\" formats.\n Previously, \"isomtime\" was the default but produced a verbose human format,\n while \"mtime\" produced a ISO-8601-like format.\n The behaviours have been swapped (so \"mtime\" is human, \"isomtime\" is ISO-like),\n and the default is now \"mtime\".\n \"isomtime\" is now a real ISO-8601 format (\"T\" between date and time, not a space).\n- create/recreate --list: file status for all files used to get announced *AFTER*\n the file (with borg < 1.2). Now, file status is announced *BEFORE* the file\n contents are processed. If the file status changes later (e.g. due to an error\n or a content change), the updated/final file status will be printed again.\n- removed deprecated-since-long stuff (deprecated since):\n\n - command \"borg change-passphrase\" (2017-02), use \"borg key ...\"\n - option \"--keep-tag-files\" (2017-01), use \"--keep-exclude-tags\"\n - option \"--list-format\" (2017-10), use \"--format\"\n - option \"--ignore-inode\" (2017-09), use \"--files-cache\" w/o \"inode\"\n - option \"--no-files-cache\" (2017-09), use \"--files-cache=disabled\"\n- removed BORG_HOSTNAME_IS_UNIQUE env var.\n to use borg you must implement one of these 2 scenarios:\n\n - 1) the combination of FQDN and result of uuid.getnode() must be unique\n and stable (this should be the case for almost everybody, except when\n having duplicate FQDN *and* MAC address or all-zero MAC address)\n - 2) if you are aware that 1) is not the case for you, you must set\n BORG_HOST_ID env var to something unique.\n- exit with 128 + signal number, #5161.\n if you have scripts expecting rc == 2 for a signal exit, you need to update\n them to check for >= 128.\n\n\n.. _changelog_1x:\n\nChange Log 1.x\n==============\n\nVersion 1.3.0a1 (2022-04-15)\n----------------------------\n\nPlease note:\n\nThis is an alpha release, only for testing - do not use this with production repos.\n\nNew features:\n\n- init: new --encryption=(repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305)\n\n - New, better, faster crypto (see encryption-aead diagram in the docs), #6463.\n - New AEAD cipher suites: AES-OCB and CHACHA20-POLY1305.\n - Session keys are derived via HKDF from random session id and master key.\n - Nonces/MessageIVs are counters starting from 0 for each session.\n - AAD: chunk id, key type, messageIV, sessionID are now authenticated also.\n - Solves the potential AES-CTR mode counter management issues of the legacy crypto.\n- init: --key-algorithm=argon2 (new default KDF, older pbkdf2 also still available)\n\n borg key change-passphrase / change-location keeps the key algorithm unchanged.\n- key change-algorithm: to upgrade existing keys to argon2 or downgrade to pbkdf2.\n\n We recommend you to upgrade unless you have to keep the key compatible with older versions of borg.\n- key change-location: usable for repokey <-> keyfile location change\n- benchmark cpu: display benchmarks of cpu bound stuff\n- export-tar: new --tar-format=PAX (default: GNU)\n- import-tar/export-tar: can use PAX format for ctime and atime support\n- import-tar/export-tar: --tar-format=BORG: roundtrip ALL item metadata, #5830\n- repository: create and use version 2 repos only for now\n- repository: implement PUT2: header crc32, overall xxh64, #1704\n\nOther changes:\n\n- require python >= 3.9, #6315\n- simplify libs setup, #6482\n- unbundle most bundled 3rd party code, use libs, #6316\n- use libdeflate.crc32 (Linux and all others) or zlib.crc32 (macOS)\n- repository: code cleanups / simplifications\n- internal crypto api: speedups / cleanups / refactorings / modernisation\n- remove \"borg upgrade\" support for \"attic backup\" repos\n- remove PassphraseKey code and borg key migrate-to-repokey command\n- OpenBSD: build borg with OpenSSL (not: LibreSSL), #6474\n- remove support for LibreSSL, #6474\n- remove support for OpenSSL < 1.1.1\n\n\nVersion 1.2.7 (2023-12-02)\n--------------------------\n\nFor upgrade and compatibility hints, please also read the section \"Upgrade Notes\"\nabove.\n\nFixes:\n\n- docs: CVE-2023-36811 upgrade steps: consider checkpoint archives, #7802\n- check/compact: fix spurious reappearance of orphan chunks since borg 1.2, #6687 -\n this consists of 2 fixes:\n\n - for existing chunks: check --repair: recreate shadow index, #7897 #6687\n - for newly created chunks: update shadow index when doing a double-put, #7896 #5661\n\n If you have experienced issue #6687, you may want to run borg check --repair\n after upgrading to borg 1.2.7 to recreate the shadow index and get rid of the\n issue for existing chunks.\n- LockRoster.modify: no KeyError if element was already gone, #7937\n- create --X-from-command: run subcommands with a clean environment, #7916\n- list --sort-by: support \"archive\" as alias of \"name\", #7873\n- fix rc and msg if arg parsing throws an exception, #7885\n\nOther changes:\n\n- support and test on Python 3.12\n- include unistd.h in _chunker.c (fix for Python 3.13)\n- allow msgpack 1.0.6 and 1.0.7\n- TAM issues: show tracebacks, improve borg check logging, #7797\n- replace \"datetime.utcfromtimestamp\" with custom helper to avoid\n deprecation warnings when using Python 3.12\n- vagrant:\n\n - use generic/debian9 box, fixes #7579\n - add VM with debian bookworm / test on OpenSSL 3.0.x.\n- docs:\n\n - not only attack/unsafe, can also be a fs issue, #7853\n - point to CVE-2023-36811 upgrade steps from borg 1.1 to 1.2 upgrade steps, #7899\n - upgrade steps needed for all kinds of repos (including \"none\" encryption mode), #7813\n - upgrade steps: talk about consequences of borg check, #7816\n - upgrade steps: remove period that could be interpreted as part of the command\n - automated-local.rst: use GPT UUID for consistent udev rule\n - create disk/partition sector backup by disk serial number, #7934\n - update macOS hint about full disk access\n - clarify borg prune -a option description, #7871\n - readthedocs: also build offline docs (HTMLzip), #7835\n - frontends: add \"check.rebuild_refcounts\" message\n\n\nVersion 1.2.6 (2023-08-31)\n--------------------------\n\nFixes:\n\n- The upgrade procedure docs as published with borg 1.2.5 did not work, if the\n repository had archives resulting from a borg rename or borg recreate operation.\n\n The updated docs now use BORG_WORKAROUNDS=ignore_invalid_archive_tam at some\n places to avoid that issue, #7791.\n\n See: fix pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811),\n details and necessary upgrade procedure described above.\n\nOther changes:\n\n- updated 1.2.5 changelog entry: 1.2.5 already has the fix for rename/recreate.\n- remove cython restrictions. recommended is to build with cython 0.29.latest,\n because borg 1.2.x uses this since years and it is very stable.\n you can also try to build with cython 3.0.x, there is a good chance that it works.\n as a 3rd option, we also bundle the `*.c` files cython outputs in the release\n pypi package, so you can also just use these and not need cython at all.\n\n\nVersion 1.2.5 (2023-08-30)\n--------------------------\n\nFixes:\n\n- Security: fix pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811),\n see details and necessary upgrade procedure described above.\n- rename/recreate: correctly update resulting archive's TAM, see #7791\n- create: do not try to read parent dir of recursion root, #7746\n- extract: fix false warning about pattern never matching, #4110\n- diff: remove surrogates before output, #7535\n- compact: clear empty directories at end of compact process, #6823\n- create --files-cache=size: fix crash, #7658\n- keyfiles: improve key sanity check, #7561\n- only warn about \"invalid\" chunker params, #7590\n- ProgressIndicatorPercent: fix space computation for wide chars, #3027\n- improve argparse validator error messages\n\nNew features:\n\n- mount: make up volname if not given (macOS), #7690.\n macFUSE supports a volname mount option to give what finder displays on the\n desktop / in the directory view. if the user did not specify it, we make\n something up, because otherwise it would be \"macFUSE Volume 0 (Python)\" and\n hide the mountpoint directory name.\n- BORG_WORKAROUNDS=authenticated_no_key to extract from authenticated repos\n without key, #7700\n\nOther changes:\n\n- add `utcnow()` helper function to avoid deprecated `datetime.utcnow()`\n- stay on latest Cython 0.29 (0.29.36) for borg 1.2.x (do not use Cython 3.0 yet)\n- docs:\n\n - move upgrade notes to own section, see #7546\n - mount -olocal: how to show mount in finder's sidebar, #5321\n - list: fix --pattern examples, #7611\n - improve patterns help\n - incl./excl. options, path-from-stdin exclusiveness\n - obfuscation docs: markup fix, note about MAX_DATA_SIZE\n - --one-file-system: add macOS apfs notes, #4876\n - improve --one-file-system help string, #5618\n - rewrite borg check docs\n - improve the docs for --keep-within, #7687\n - fix borg init command in environment.rst.inc\n - 1.1.x upgrade notes: more precise borg upgrade instructions, #3396\n\n- tests:\n\n - fix repo reopen\n - avoid long ids in pytest output\n - check buzhash chunksize distribution, see #7586\n\n\nVersion 1.2.4 (2023-03-24)\n--------------------------\n\nNew features:\n\n- import-tar: add --ignore-zeros to process concatenated tars, #7432.\n- debug id-hash: computes file/chunk content id-hash, #7406\n- diff: --content-only does not show mode/ctime/mtime changes, #7248\n- diff: JSON strings in diff output are now sorted alphabetically\n\nBug fixes:\n\n- xattrs: fix namespace processing on FreeBSD, #6997\n- diff: fix path related bug seen when addressing deferred items.\n- debug get-obj/put-obj: always give chunkid as cli param, see #7290\n (this is an incompatible change, see also borg debug id-hash)\n- extract: fix mtime when ResourceFork xattr is set (macOS specific), #7234\n- recreate: without --chunker-params, do not re-chunk, #7337\n- recreate: when --target is given, do not detect \"nothing to do\".\n use case: borg recreate -a src --target dst can be used to make a copy\n of an archive inside the same repository, #7254.\n- set .hardlink_master for ALL hardlinkable items, #7175\n- locking: fix host, pid, tid order.\n tid (thread id) must be parsed as hex from lock file name.\n- update development.lock.txt, including a setuptools security fix, #7227\n\nOther changes:\n\n- requirements: allow msgpack 1.0.5 also\n- upgrade Cython to 0.29.33\n- hashindex minor fixes, refactor, tweaks, tests\n- use os.replace not os.rename\n- remove BORG_LIBB2_PREFIX (not used any more)\n- docs:\n\n - BORG_KEY_FILE: clarify docs, #7444\n - update FAQ about locale/unicode issues, #6999\n - improve mount options rendering, #7359\n - make timestamps in manual pages reproducible\n - installation: update Fedora in distribution list, #7357\n- tests:\n\n - fix test_size_on_disk_accurate for large st_blksize, #7250\n - add same_ts_ns function and use it for relaxed timestamp comparisons\n - \"auto\" compressor tests: don't assume a specific size,\n do not assume zlib is better than lz4, #7363\n - add test for extracted directory mtime\n- vagrant:\n\n - upgrade local freebsd 12.1 box -> generic/freebsd13 box (13.1)\n - use pythons > 3.8 which work on freebsd 13.1\n - pyenv: also install python 3.11.1 for testing\n - pyenv: use python 3.10.1, 3.10.0 build is broken on freebsd\n\n\nVersion 1.2.3 (2022-12-24)\n--------------------------\n\nFixes:\n\n- create: fix --list --dry-run output for directories, #7209\n- diff/recreate: normalize chunker params before comparing them, #7079\n- check: fix uninitialised variable if repo is completely empty, #7034\n- xattrs: improve error handling, #6988\n- fix args.paths related argparsing, #6994\n- archive.save(): always use metadata from stats (e.g. nfiles, size, ...), #7072\n- tar_filter: recognize .tar.zst as zstd, #7093\n- get_chunker: fix missing sparse=False argument, #7056\n- file_integrity.py: make sure file_fd is always closed on exit\n- repository: cleanup(): close segment before unlinking\n- repository: use os.replace instead of os.rename\n\nOther changes:\n\n- remove python < 3.7 compatibility code\n- do not use version_tuple placeholder in setuptools_scm template\n- CI: fix tox4 passenv issue, #7199\n- vagrant: update to python 3.9.16, use the openbsd 7.1 box\n- misc. test suite and docs fixes / improvements\n- remove deprecated --prefix from docs, #7109\n- Windows: use MSYS2 for Github CI, remove Appveyor CI\n\n\nVersion 1.2.2 (2022-08-20)\n--------------------------\n\nNew features:\n\n- prune/delete --checkpoint-interval=1800 and ctrl-c/SIGINT support, #6284\n\nFixes:\n\n- SaveFile: use a custom mkstemp with mode support, #6933, #6400, #6786.\n This fixes umask/mode/ACL issues (and also \"chmod not supported\" exceptions\n seen in 1.2.1) of files updated using SaveFile, e.g. the repo config.\n- hashindex_compact: fix eval order (check idx before use), #5899\n- create --paths-from-(stdin|command): normalize paths, #6778\n- secure_erase: avoid collateral damage, #6768.\n If a hardlink copy of a repo was made and a new repo config shall be saved,\n do NOT fill in random garbage before deleting the previous repo config,\n because that would damage the hardlink copy.\n- list: fix {flags:} formatting, #6081\n- check: try harder to create the key, #5719\n- misc commands: ctrl-c must not kill other subprocesses, #6912\n\n - borg create with a remote repo via ssh\n - borg create --content-from-command\n - borg create --paths-from-command\n - (de)compression filter process of import-tar / export-tar\n\nOther changes:\n\n- deprecate --prefix, use -a / --glob-archives, see #6806\n- make setuptools happy (\"package would be ignored\"), #6874\n- fix pyproject.toml to create a fixed _version.py file, compatible with both\n old and new setuptools_scm version, #6875\n- automate asciinema screencasts\n- CI: test on macOS 12 without fuse / fuse tests\n (too troublesome on github CI due to kernel extensions needed by macFUSE)\n- tests: fix test_obfuscate byte accounting\n- repository: add debug logging for issue #6687\n- _chunker.c: fix warnings on macOS\n- requirements.lock.txt: use the latest cython 0.29.32\n- docs:\n\n - add info on man page installation, #6894\n - update archive_progress json description about \"finished\", #6570\n - json progress_percent: some values are optional, #4074\n - FAQ: full quota / full disk, #5960\n - correct shell syntax for installation using git\n\n\nVersion 1.2.1 (2022-06-06)\n--------------------------\n\nFixes:\n\n- create: skip with warning if opening the parent dir of recursion root fails, #6374\n- create: fix crash. metadata stream can produce all-zero chunks, #6587\n- fix crash when computing stats, escape % chars in archive name, #6500\n- fix transaction rollback: use files cache filename as found in txn.active/, #6353\n- import-tar: kill filter process in case of borg exceptions, #6401 #6681\n- import-tar: fix mtime type bug\n- ensure_dir: respect umask for created directory modes, #6400\n- SaveFile: respect umask for final file mode, #6400\n- check archive: improve error handling for corrupt archive metadata block, make\n robust_iterator more robust, #4777\n- pre12-meta cache: do not use the cache if want_unique is True, #6612\n- fix scp-style repo url parsing for ip v6 address, #6526\n- mount -o versions: give clear error msg instead of crashing.\n it does not make sense to request versions view if you only look at 1 archive,\n but the code shall not crash in that case as it did, but give a clear error msg.\n- show_progress: add finished=true/false to archive_progress json, #6570\n- delete/prune: fix --iec mode output (decimal vs. binary units), #6606\n- info: fix authenticated mode repo to show \"Encrypted: No\", #6462\n- diff: support presence change for blkdev, chrdev and fifo items, #6615\n\nNew features:\n\n- delete: add repository id and location to prompt, #6453\n- borg debug dump-repo-objs --ghost: new --segment=S --offset=O options\n\nOther changes:\n\n- support python 3.11\n- allow msgpack 1.0.4, #6716\n- load_key: no key is same as empty key, #6441\n- give a more helpful error msg for unsupported key formats, #6561\n- better error msg for defect or unsupported repo configs, #6566\n- docs:\n\n - document borg 1.2 pattern matching behavior change, #6407\n Make clear that absolute paths always go into the matcher as if they are\n relative (without leading slash). Adapt all examples accordingly.\n - authentication primitives: improved security and performance infos\n - mention BORG_FILES_CACHE_SUFFIX as alternative to BORG_FILES_CACHE_TTL, #5602\n - FAQ: add a hint about --debug-topic=files_cache\n - improve borg check --max-duration description\n - fix values of TAG bytes, #6515\n - borg compact --cleanup-commits also runs a normal compaction, #6324\n - virtualization speed tips\n - recommend umask for passphrase file perms\n - borg 1.2 is security supported\n - update link to ubuntu packages, #6485\n - use --numeric-ids in pull mode docs\n - remove blake2 docs, blake2 code not bundled any more, #6371\n - clarify on-disk order and size of segment file log entry fields, #6357\n - docs building: do not transform --/--- to unicode dashes\n- tests:\n\n - check that borg does not require pytest for normal usage, fixes #6563\n - fix OpenBSD symlink mode test failure, #2055\n- vagrant:\n\n - darwin64: remove fakeroot, #6314\n - update development.lock.txt\n - use pyinstaller 4.10 and python 3.9.13 for binary build\n - upgrade VMCPUS and xdistn from 4 to 16, maybe this speeds up the tests\n- crypto:\n\n - use hmac.compare_digest instead of ==, #6470\n - hmac_sha256: replace own cython wrapper code by hmac.digest python stdlib (since py38)\n - hmac and blake2b minor optimizations and cleanups\n - removed some unused crypto related code, #6472\n - avoid losing the key (potential use-after-free). this never could happen in\n 1.2 due to the way we use the code. The issue was discovered in master after\n other changes, so we also \"fixed\" it here before it bites us.\n- setup / build:\n\n - add pyproject.toml, fix sys.path, #6466\n - setuptools_scm: also require it via pyproject.toml\n - allow extra compiler flags for every extension build\n - fix misc. C / Cython compiler warnings, deprecation warnings\n - fix zstd.h include for bundled zstd, #6369\n- source using python 3.8 features: ``pyupgrade --py38-plus ./**/*.py``\n\n\nVersion 1.2.0 (2022-02-22 22:02:22 :-)\n--------------------------------------\n\nPlease note:\n\nThis is the first borg 1.2 release, so be careful and read the notes below.\n\nUpgrade notes:\n\nStrictly taken, nothing special is required for upgrading to 1.2, but some\nthings can be recommended:\n\n- do you already want to upgrade? 1.1.x also will get fixes for a while.\n- be careful, first upgrade your less critical / smaller repos.\n- first upgrade to a recent 1.1.x release - especially if you run some older\n 1.1.* or even 1.0.* borg release.\n- using that, run at least one `borg create` (your normal backup), `prune`\n and especially a `check` to see everything is in a good state.\n- check the output of `borg check` - if there is anything special, consider\n a `borg check --repair` followed by another `borg check`.\n- if everything is fine so far (borg check reports no issues), you can consider\n upgrading to 1.2.0. if not, please first fix any already existing issue.\n- if you want to play safer, first **create a backup of your borg repository**.\n- upgrade to latest borg 1.2.x release (you could use the fat binary from\n github releases page)\n- run `borg compact --cleanup-commits` to clean up a ton of 17 bytes long files\n in your repo caused by a borg 1.1 bug\n- run `borg check` again (now with borg 1.2.x) and check if there is anything\n special.\n- run `borg info` (with borg 1.2.x) to build the local pre12-meta cache (can\n take significant time, but after that it will be fast) - for more details\n see below.\n- check the compatibility notes (see below) and adapt your scripts, if needed.\n- if you run into any issues, please check the github issue tracker before\n posting new issues there or elsewhere.\n\nIf you follow this procedure, you can help avoiding that we get a lot of\n\"borg 1.2\" issue reports that are not really 1.2 issues, but existed before\nand maybe just were not noticed.\n\nCompatibility notes:\n\n- matching of path patterns has been aligned with borg storing relative paths.\n Borg archives file paths without leading slashes. Previously, include/exclude\n patterns could contain leading slashes. You should check your patterns and\n remove leading slashes.\n- dropped support / testing for older Pythons, minimum requirement is 3.8.\n In case your OS does not provide Python >= 3.8, consider using our binary,\n which does not need an external Python interpreter. Or continue using\n borg 1.1.x, which is still supported.\n- freeing repository space only happens when \"borg compact\" is invoked.\n- mount: the default for --numeric-ids is False now (same as borg extract)\n- borg create --noatime is deprecated. Not storing atime is the default behaviour\n now (use --atime if you want to store the atime).\n- list: corrected mix-up of \"isomtime\" and \"mtime\" formats.\n Previously, \"isomtime\" was the default but produced a verbose human format,\n while \"mtime\" produced a ISO-8601-like format.\n The behaviours have been swapped (so \"mtime\" is human, \"isomtime\" is ISO-like),\n and the default is now \"mtime\".\n \"isomtime\" is now a real ISO-8601 format (\"T\" between date and time, not a space).\n- create/recreate --list: file status for all files used to get announced *AFTER*\n the file (with borg < 1.2). Now, file status is announced *BEFORE* the file\n contents are processed. If the file status changes later (e.g. due to an error\n or a content change), the updated/final file status will be printed again.\n- removed deprecated-since-long stuff (deprecated since):\n\n - command \"borg change-passphrase\" (2017-02), use \"borg key ...\"\n - option \"--keep-tag-files\" (2017-01), use \"--keep-exclude-tags\"\n - option \"--list-format\" (2017-10), use \"--format\"\n - option \"--ignore-inode\" (2017-09), use \"--files-cache\" w/o \"inode\"\n - option \"--no-files-cache\" (2017-09), use \"--files-cache=disabled\"\n- removed BORG_HOSTNAME_IS_UNIQUE env var.\n to use borg you must implement one of these 2 scenarios:\n\n - 1) the combination of FQDN and result of uuid.getnode() must be unique\n and stable (this should be the case for almost everybody, except when\n having duplicate FQDN *and* MAC address or all-zero MAC address)\n - 2) if you are aware that 1) is not the case for you, you must set\n BORG_HOST_ID env var to something unique.\n- exit with 128 + signal number, #5161.\n if you have scripts expecting rc == 2 for a signal exit, you need to update\n them to check for >= 128.\n\nFixes:\n\n- diff: reduce memory consumption, fix is_hardlink_master, #6295\n- compact: fix / improve freeable / freed space log output\n\n - derive really freed space from quota use before/after, #5679\n - do not say \"freeable\", but \"maybe freeable\" (based on hint, unsure)\n- fix race conditions in internal SaveFile function, #6306 #6028\n- implement internal safe_unlink (was: truncate_and_unlink) function more safely:\n usually it does not truncate any more, only under \"disk full\" circumstances\n and only if there is only one hardlink.\n see: https://github.com/borgbackup/borg/discussions/6286\n\nOther changes:\n\n- info: use a pre12-meta cache to accelerate stats for borg < 1.2 archives.\n the first time borg info is invoked on a borg 1.1 repo, it can take a\n rather long time computing and caching some stats values for 1.1 archives,\n which borg 1.2 archives have in their archive metadata structure.\n be patient, esp. if you have lots of old archives.\n following invocations are much faster due to the cache.\n related change: add archive name to calc_stats progress display.\n- docs:\n\n - add borg 1.2 upgrade notes, #6217\n - link to borg placeholders and borg patterns help\n - init: explain the encryption modes better\n - clarify usage of patternfile roots\n - put import-tar docs into same file as export-tar docs\n - explain the difference between a path that ends with or without a slash,\n #6297\n\n\nVersion 1.2.0rc1 (2022-02-05)\n-----------------------------\n\nFixes:\n\n- repo::archive location placeholder expansion fixes, #5826, #5998\n- repository: fix intermediate commits, shall be at end of current segment\n- delete: don't commit if nothing was deleted, avoid cache sync, #6060\n- argument parsing: accept some options only once, #6026\n- disallow overwriting of existing keyfiles on init, #6036\n- if ensure_dir() fails, give more informative error message, #5952\n\nNew features:\n\n- delete --force: do not ask when deleting a repo, #5941\n\nOther changes:\n\n- requirements: exclude broken or incompatible-with-pyinstaller setuptools\n- add a requirements.d/development.lock.txt and use it for vagrant\n- tests:\n\n - added nonce-related tests\n - refactor: remove assert_true\n - vagrant: macos box tuning, netbsd box fixes, #5370, #5922\n- docs:\n\n - update install docs / requirements docs, #6180\n - borg mount / FUSE \"versions\" view is not experimental any more\n - --pattern* is not experimental any more, #6134\n - impact of deleting path/to/repo/nonce, #5858\n - key export: add examples, #6204\n - ~/.config/borg/keys is not used for repokey keys, #6107\n - excluded parent dir's metadata can't restore\n\n\nVersion 1.2.0b4 (2022-01-23)\n----------------------------\n\nFixes:\n\n- create: fix passing device nodes and symlinks to --paths-from-stdin, #6009\n- create --dry-run: fix display of kept tagfile, #5834\n- check --repair: fix missing parameter in \"did not consistently fail\" msg, #5822\n- fix hardlinkable file type check, #6037\n- list: remove placeholders for shake_* hashes, #6082\n- prune: handle case of calling prune_split when there are no archives, #6015\n- benchmark crud: make sure cleanup of borg-test-data files/dir happens, #5630\n- do not show archive name in repository-related error msgs, #6014\n- prettier error msg (no stacktrace) if exclude file is missing, #5734\n- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979\n- fix pyinstaller detection for dir-mode, #5897\n- atomically create the CACHE_TAG file, #6028\n- deal with the SaveFile/SyncFile race, docs, see #6056 708a5853\n- avoid expanding path into LHS of formatting operation + tests, #6064 #6063\n- repository: quota / compactable computation fixes\n- info: emit repo info even if repo has 0 archives + test, #6120\n\nNew features:\n\n- check --repair: significantly speed up search for next valid object in segment, #6022\n- check: add progress indicator for archive check, #5809\n- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024\n- create: allow --files-cache=size (this is potentially dangerous, use on your own risk), #5686\n- import-tar: implement import-tar to complement export-tar, #2233\n- implement BORG_SELFTEST env variable (can be carefully used to speedup borg hosting), #5871\n- key export: print key if path is '-' or not given, #6092\n- list --format: Add command_line to format keys\n\nOther changes:\n\n- pypi metadata: alpha -> beta\n- require python 3.8+, #5975\n- use pyinstaller 4.7\n- allow msgpack 1.0.3\n- upgrade to bundled xxhash to 0.8.1\n- import-tar / export-tar: tar file related changes:\n\n - check for short tarfile extensions\n - add .lz4 and .zstd\n - fix docs about extensions and decompression commands\n- add github codeql analysis, #6148\n- vagrant:\n\n - box updates / add new boxes / remove outdated and broken boxes\n - use Python 3.9.10 (incl. binary builds) and 3.10.0\n - fix pyenv initialisation, #5798\n - fix vagrant scp on macOS, #5921\n - use macfuse instead of osxfuse\n- shell completions:\n\n - update shell completions to 1.1.17, #5923\n - remove BORG_LIBC completion, since 9914968 borg no longer uses find_library().\n- docs:\n\n - fixed readme.rst irc webchat link (we use libera chat now, not freenode)\n - fix exceptions thrown by `setup.py build_man`\n - check --repair: recommend checking hw before check --repair, #5855\n - check --verify-data: clarify and document conflict with --repository-only, #5808\n - serve: improve ssh forced commands docs, #6083\n - list: improve docs for `borg list` --format, #6061\n - list: remove --list-format from borg list\n - FAQ: fix manifest-timestamp path (inside security dir)\n - fix the broken link to .nix file\n - document behavior for filesystems with inconsistent inodes, #5770\n - clarify user_id vs uid for fuse, #5723\n - clarify pattern usage with commands, #5176\n - clarify pp vs. pf pattern type, #5300\n - update referenced freebsd/macOS versions used for binary build, #5942\n - pull mode: add some warnings, #5827\n - clarify \"you will need key and passphrase\" borg init warning, #4622\n - add missing leading slashes in help patterns, #5857\n - add info on renaming repositories, #5240\n - check: add notice about defective hardware, #5753\n - mention tar --compare (compare archive to fs files), #5880\n - add note about grandfather-father-son backup retention policy / rotation scheme, #6006\n - permissions note rewritten to make it less confusing\n - create github security policy\n - remove leftovers of BORG_HOSTNAME_IS_UNIQUE\n - excluded parent dir's metadata can't restore. (#6062)\n - if parent dir is not extracted, we do not have its metadata\n - clarify who starts the remote agent\n\n\nVersion 1.2.0b3 (2021-05-12)\n----------------------------\n\nFixes:\n\n- create: fix --progress --log-json, #4360#issuecomment-774580052\n- do not load files cache for commands not using it, #5673\n- fix repeated cache tag file writing bug\n\nNew features:\n\n- create/recreate: print preliminary file status early, #5417\n- create/extract: add --noxattrs and --noacls options, #3955\n- create: verbose files cache logging via --debug-topic=files_cache, #5659\n- mount: implement --numeric-ids (default: False!), #2377\n- diff: add --json-lines option\n- info / create --stats: add --iec option to print sizes in powers of 1024.\n\nOther changes:\n\n- create: add --upload-(ratelimit|buffer), deprecate --remote-* options, #5611\n- create/extract/mount: add --numeric-ids, deprecate --numeric-owner option, #5724\n- config: accept non-int value for max_segment_size / storage_quota\n- use PyInstaller v4.3, #5671\n- vagrant: use Python 3.9.5 to build binaries\n- tox.ini: modernize and enable execution without preinstalling deps\n- cleanup code style checks\n- get rid of distutils, use setuptools+packaging\n- github CI: test on Python 3.10-dev\n- check: missing / healed chunks: always tell chunk ID, #5704\n- docs:\n\n - remove bad /var/cache exclusion in example commands, #5625\n - misc. fixes and improvements, esp. for macOS\n - add unsafe workaround to use an old repo copy, #5722\n\n\nVersion 1.2.0b2 (2021-02-06)\n----------------------------\n\nFixes:\n\n- create: do not recurse into duplicate roots, #5603\n- create: only print stats if not ctrl-c'ed, fixes traceback, #5668\n- extract:\n improve exception handling when setting xattrs, #5092.\n emit a warning message giving the path, xattr key and error message.\n continue trying to restore other xattrs and bsdflags of the same file\n after an exception with xattr-setting happened.\n- export-tar:\n fix memory leak with ssh: remote repository, #5568.\n fix potential memory leak with ssh: remote repository with partial extraction.\n- remove empty shadowed_segments lists, #5275\n- fix bad default: manifest.archives.list(consider_checkpoints=False),\n fixes tracebacks / KeyErros for missing objects in ChunkIndex, #5668\n\nNew features:\n\n- create: improve sparse file support\n\n - create --sparse (detect sparse file holes) and file map support,\n only for the \"fixed\" chunker, #14\n - detect all-zero chunks in read data in \"buzhash\" and \"fixed\" chunkers\n - cached_hash: use a small LRU cache to accelerate all-zero chunks hashing\n - use cached_hash also to generate all-zero replacement chunks\n- create --remote-buffer, add a upload buffer for remote repos, #5574\n- prune: keep oldest archive when retention target not met\n\nOther changes:\n\n- use blake2 from python 3.6+ hashlib\n (this removes the requirement for libb2 and the bundled blake2 code)\n- also accept msgpack up to 1.0.2.\n exclude 1.0.1 though, which had some issues (not sure they affect borg).\n- create: add repository location to --stats output, #5491\n- check: debug log the segment filename\n- delete: add a --list switch to borg delete, #5116\n- borg debug dump-hints - implemented to e.g. to look at shadow_index\n- Tab completion support for additional archives for 'borg delete'\n- refactor: have one borg.constants.zero all-zero bytes object\n- refactor shadow_index updating repo.put/delete, #5661, #5636.\n- docs:\n\n - add another case of attempted hardlink usage\n - fix description of borg upgrade hardlink usage, #5518\n - use HTTPS everywhere\n - add examples for --paths-from-stdin, --paths-from-command, --paths-separator, #5644\n - fix typos/grammar\n - update docs for dev environment installation instructions\n - recommend running tests only on installed versions for setup\n - add badge with current status of package\n- vagrant:\n\n - use brew install --cask ..., #5557\n - use Python 3.9.1 and PyInstaller 4.1 to build the borg binary\n\n\nVersion 1.2.0b1 (2020-12-06)\n----------------------------\n\nFixes:\n\n- BORG_CACHE_DIR crashing borg if empty, atomic handling of\n recursive directory creation, #5216\n- fix --dry-run and --stats coexistence, #5415\n- allow EIO with warning when trying to hardlink, #4336\n- export-tar: set tar format to GNU_FORMAT explicitly, #5274\n- use --timestamp for {utcnow} and {now} if given, #5189\n- make timestamp helper timezone-aware\n\nNew features:\n\n- create: implement --paths-from-stdin and --paths-from-command, see #5492.\n These switches read paths to archive from stdin. Delimiter can specified\n by --paths-delimiter=DELIM. Paths read will be added honoring every\n option but exclusion options and --one-file-system. borg won't recurse\n into directories.\n- 'obfuscate' pseudo compressor obfuscates compressed chunk size in repo\n- add pyfuse3 (successor of llfuse) as an alternative lowlevel fuse\n implementation to llfuse (deprecated), #5407.\n FUSE implementation can be switched via env var BORG_FUSE_IMPL.\n- allow appending to the files cache filename with BORG_FILES_CACHE_SUFFIX\n- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333\n\nOther changes:\n\n- split recursive directory walking/processing into directory walking and\n item processing.\n- fix warning by importing setuptools before distutils.\n- debug info: include infos about FUSE implementation, #5546\n- testing:\n\n - add a test for the hashindex corruption bug, #5531 #4829\n - move away from travis-ci, use github actions, #5528 #5467\n - test both on fuse2 and fuse3\n - upload coverage reports to codecov\n - fix spurious failure in test_cache_files, #5438\n - add tests for Location.with_timestamp\n - tox: add a non-fuse env to the envlist\n- vagrant:\n\n - use python 3.7.latest and pyinstaller 4.0 for binary creation\n - pyinstaller: compute basepath from spec file location\n - vagrant: updates/fixes for archlinux box, #5543\n- docs:\n\n - \"filename with spaces\" example added to exclude file, #5236\n - add a hint about sleeping computer, #5301\n - how to adjust macOS >= Catalina security settings, #5303\n - process/policy for adding new compression algorithms\n - updated docs about hacked backup client, #5480\n - improve ansible deployment docs, make it more generic\n - how to approach borg speed issues, give speed example, #5371\n - fix mathematical inaccuracy about chunk size, #5336\n - add example for excluding content using --pattern cli option\n - clarify borg create's '--one-file-system' option, #4009\n - improve docs/FAQ about append-only remote repos, #5497\n - fix reST markup issues, labels\n - add infos about contributor retirement status\n\n\nVersion 1.2.0a9 (2020-10-05)\n----------------------------\n\nFixes:\n\n- fix memory leak related to preloading, #5202\n- check --repair: fix potential data loss, #5325\n- persist shadow_index in between borg runs, #4830\n- fix hardlinked CACHEDIR.TAG processing, #4911\n- --read-special: .part files also should be regular files, #5217\n- allow server side enforcing of umask, --umask is for the local borg\n process only (see docs), #4947\n- exit with 128 + signal number, #5161\n- borg config --list does not show last_segment_checked, #5159\n- locking:\n\n - fix ExclusiveLock race condition bug, #4923\n - fix race condition in lock migration, #4953\n - fix locking on openindiana, #5271\n\nNew features:\n\n- --content-from-command: create archive using stdout of given command, #5174\n- allow key-import + BORG_KEY_FILE to create key files\n- build directory-based binary for macOS to avoid Gatekeeper delays\n\nOther changes:\n\n- upgrade bundled zstd to 1.4.5\n- upgrade bundled xxhash to 0.8.0, #5362\n- if self test fails, also point to OS and hardware, #5334\n- misc. shell completions fixes/updates, rewrite zsh completion\n- prettier error message when archive gets too big, #5307\n- stop relying on `false` exiting with status code 1\n- rephrase some warnings, #5164\n- parseformat: unnecessary calls removed, #5169\n- testing:\n\n - enable Python3.9 env for test suite and VMs, #5373\n - drop python 3.5, #5344\n - misc. vagrant fixes/updates\n - misc. testing fixes, #5196\n- docs:\n\n - add ssh-agent pull backup method to doc, #5288\n - mention double --force in prune docs\n - update Homebrew install instructions, #5185\n - better description of how cache and rebuilds of it work\n and how the workaround applies to that\n - point to borg create --list item flags in recreate usage, #5165\n - add a note to create from stdin regarding files cache, #5180\n - add security faq explaining AES-CTR crypto issues, #5254\n - clarify --exclude-if-present in recreate, #5193\n - add socat pull mode, #5150, #900\n - move content of resources doc page to community project, #2088\n - explain hash collision, #4884\n - clarify --recompress option, #5154\n\n\nVersion 1.2.0a8 (2020-04-22)\n----------------------------\n\nFixes:\n\n- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829.\n Please read and follow the more detailed notes close to the top of this document.\n- fix crash when upgrading erroneous hints file, #4922\n- commit-time free space calc: ignore bad compact map entries, #4796\n- info: if the archive doesn't exist, print a pretty message, #4793\n- --prefix / -P: fix processing, avoid argparse issue, #4769\n- ignore EACCES (errno 13) when hardlinking, #4730\n- add a try catch when formatting the info string, #4818\n- check: do not stumble over invalid item key, #4845\n- update prevalence of env vars to set config and cache paths\n- mount: fix FUSE low linear read speed on large files, #5032\n- extract: fix confusing output of borg extract --list --strip-components, #4934\n- recreate: support --timestamp option, #4745\n- fix ProgressIndicator msgids (JSON output), #4935\n- fuse: set f_namemax in statfs result, #2684\n- accept absolute paths on windows\n- pyinstaller: work around issue with setuptools > 44\n\nNew features:\n\n- chunker speedup (plus regression test)\n- added --consider-checkpoints and related test, #4788\n- added --noflags option, deprecate --nobsdflags option, #4489\n- compact: add --threshold option, #4674\n- mount: add birthtime to FUSE entries\n- support platforms with no os.link, #4901 - if we don't have os.link,\n we just extract another copy instead of making a hardlink.\n- move sync_file_range to its own extension for better platform compatibility.\n- new --bypass-lock option to bypass locking, e.g. for read-only repos\n- accept absolute paths by removing leading slashes in patterns of all\n sorts but re: style, #4029\n- delete: new --keep-security-info option\n\nOther changes:\n\n- support msgpack 0.6.2 and 1.0.0, #5065\n- upgrade bundled zstd to 1.4.4\n- upgrade bundled lz4 to 1.9.2\n- upgrade xxhash to 0.7.3\n- require recent enough llfuse for birthtime support, #5064\n- only store compressed data if the result actually is smaller, #4516\n- check: improve error output for matching index size, see #4829\n- ignore --stats when given with --dry-run, but continue, #4373\n- replaced usage of os.statvfs with shutil.disk_usage (better cross-platform support).\n- fuse: remove unneeded version check and compat code, micro opts\n- docs:\n\n - improve description of path variables\n - document how to delete data completely, #2929\n - add FAQ about Borg config dir, #4941\n - add docs about errors not printed as JSON, #4073\n - update usage_general.rst.inc\n - added \"Will move with BORG_CONFIG_DIR variable unless specified.\" to BORG_SECURITY_DIR info.\n - put BORG_SECURITY_DIR immediately below BORG_CONFIG_DIR (and moved BORG_CACHE_DIR up before them).\n - add paragraph regarding cache security assumptions, #4900\n - tell about borg cache security precautions\n - add FAQ describing difference between a local repo vs. repo on a server.\n - document how to test exclusion patterns without performing an actual backup\n - create: tell that \"Calculating size\" time and space needs are caused by --progress\n - fix/improve documentation for @api decorator, #4674\n - add a pull backup / push restore how-to, #1552\n - fix man pages creation, #4752\n - more general FAQ for backup and retain original paths, #4532\n - explain difference between --exclude and --pattern, #4118\n - add FAQ for preventing SSH timeout in extract, #3866\n - improve password FAQ (decrease pw length, add -w 0 option to base64 to prevent line wrap), #4591\n - add note about patterns and stored paths, #4160\n - add upgrade of tools to pip installation how-to, #5090\n - document one cause of orphaned chunks in check command, #2295\n - clean up the whole check usage paragraph\n - FAQ: linked recommended restrictions to ssh public keys on borg servers, #4946\n - fixed \"doc downplays severity of Nonce reuse issue\", #4883\n - borg repo restore instructions needed, #3428\n - new FAQ: A repo is corrupt and must be replaced with an older repo.\n - clarify borg init's encryption modes\n- native windows port:\n\n - update README_WINDOWS.rst\n - updated pyinstaller spec file to support windows builds\n- testing / CI:\n\n - improved travis config / install script, improved macOS builds\n - allow osx builds to fail, #4955\n - Windows 10 build on Appveyor CI\n- vagrant:\n\n - upgrade pyinstaller to v3.5 + patch\n - use py369 for binary build, add py380 for tests\n - fix issue in stretch VM hanging at grub installation\n - add a debian buster and a ubuntu focal VM\n - update darwin box to 10.12\n - upgrade FreeBSD box to 12.1\n - fix debianoid virtualenv packages\n - use pyenv in freebsd64 VM\n - remove the flake8 test\n - darwin: avoid error if pkg is already installed\n - debianoid: don't interactively ask questions\n\n\nVersion 1.2.0a7 (2019-09-07)\n----------------------------\n\nFixes:\n\n- slave hardlinks extraction issue, see #4350\n- extract: fix KeyError for \"partial\" extraction, #4607\n- preload chunks for hardlink slaves w/o preloaded master, #4350\n- fix preloading for old remote servers, #4652\n- fix partial extract for hardlinked contentless file types, #4725\n- Repository.open: use stat() to check for repo dir, #4695\n- Repository.check_can_create_repository: use stat() to check, ~ #4695.\n- SecurityManager.known(): check all files, #4614\n- after double-force delete, warn about necessary repair, #4704\n- cope with ANY error when importing pytest into borg.testsuite, #4652\n- fix invalid archive error message\n- setup.py: fix detection of missing Cython\n- filter out selinux xattrs, #4574\n- location arg - should it be optional? #4541\n- enable placeholder usage in --comment, #4559\n- use whitelist approach for borg serve, #4097\n\nNew features:\n\n- minimal native Windows support, see windows readme (work in progress)\n- create: first ctrl-c (SIGINT) triggers checkpoint and abort, #4606\n- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710\n- remove WSL autodetection. if WSL still has this problem, you need to\n set BORG_WORKAROUNDS=basesyncfile in the borg process environment to\n work around it.\n- support xxh64 checksum in addition to the hashlib hashes in borg list\n- enable placeholder usage in all extra archive arguments\n- enable placeholder usage in --comment, #4559\n- enable placeholder usage in --glob-archives, #4495\n- ability to use a system-provided version of \"xxhash\"\n- create:\n\n - changed the default behaviour not to store the atime of fs items. atime is\n often rather not interesting and fragile - it easily changes even if nothing\n else has changed and, if stored into the archive, spoils deduplication of\n the archive metadata stream.\n - if you give the --noatime option, borg will output a deprecation warning\n because it is currently ignored / does nothing.\n Please remove the --noatime option when using borg 1.2.\n - added a --atime option for storing files' atime into an archive\n\nOther changes:\n\n- argparser: always use REPOSITORY in metavar\n- do not check python/libc for borg serve, #4483\n- small borg compact improvements, #4522\n- compact: log freed space at INFO level\n- tests:\n\n - tox / travis: add testing on py38-dev\n - fix broken test that relied on improper zlib assumptions\n - pure-py msgpack warning shall not make a lot of tests fail, #4558\n - rename test_mount_hardlinks to test_fuse_mount_hardlinks (master)\n - vagrant: add up-to-date openindiana box (py35, openssl10)\n - get rid of confusing coverage warning, #2069\n- docs:\n\n - reiterate that 'file cache names are absolute' in FAQ,\n mention bind mount solution, #4738\n - add restore docs, #4670\n - updated docs to cover use of temp directory on remote, #4545\n - add a push-style example to borg-create(1), #4613\n - timestamps in the files cache are now usually ctime, #4583\n - benchmark crud: clarify that space is used until compact\n - update documentation of borg create,\n corrects a mention of borg 1.1 as a future version.\n - fix osxfuse github link in installation docs\n - how to supply a passphrase, use crypto devices, #4549\n - extract: document limitation \"needs empty destination\", #4598\n - update macOS Brew link\n - add note about software for automating backup\n - compact: improve docs,\n - README: new URL for funding options\n\n\nVersion 1.2.0a6 (2019-04-22)\n----------------------------\n\nFixes:\n\n- delete / prune: consider part files correctly for stats, #4507\n- fix \"all archives\" stats considering part files, #4329\n- create: only run stat_simple_attrs() once\n- create: --stats does not work with --dry-run, exit with error msg, #4373\n- give \"invalid repo\" error msg if repo config not found, #4411\n\nNew features:\n\n- display msgpack version as part of sysinfo (e.g. in tracebacks)\n\nOther changes:\n\n- docs:\n\n - sdd \"SSH Configuration\" section, #4493, #3988, #636, #4485\n - better document borg check --max-duration, #4473\n - sorted commands help in multiple steps, #4471\n- testing:\n\n - travis: use py 3.5.3 and 3.6.7 on macOS to get a pyenv-based python\n build with openssl 1.1\n - vagrant: use py 3.5.3 and 3.6.8 on darwin64 VM to build python and\n borg with openssl 1.1\n - pytest: -v and default XDISTN to 1, #4481\n\n\nVersion 1.2.0a5 (2019-03-21)\n----------------------------\n\nFixes:\n\n- warn if a file has changed while being backed up, #1750\n- lrucache: regularly remove old FDs, #4427\n- borg command shall terminate with rc 2 for ImportErrors, #4424\n- make freebsd xattr platform code api compatible with linux, #3952\n\nOther changes:\n\n- major setup code refactoring (especially how libraries like openssl, liblz4,\n libzstd, libb2 are discovered and how it falls back to code bundled with\n borg), new: uses pkg-config now (and needs python \"pkgconfig\" package\n installed), #1925\n\n if you are a borg package maintainer, please try packaging this\n (see comments in setup.py).\n- Vagrantfile: add zstd, reorder, build env vars, #4444\n- travis: install script improvements\n- update shell completions\n- docs:\n\n - add a sample logging.conf in docs/misc, #4380\n - fix spelling errors\n - update requirements / install docs, #4374\n\n\nVersion 1.2.0a4 (2019-03-11)\n----------------------------\n\nFixes:\n\n- do not use O_NONBLOCK for special files, like FIFOs, block and char devices\n when using --read-special. fixes backing up FIFOs. fixes to test. #4394\n- more LibreSSL build fixes: LibreSSL has HMAC_CTX_free and HMAC_CTX_new\n\nNew features:\n\n- check: incremental repo check (only checks crc32 for segment entries), #1657\n borg check --repository-only --max-duration SECONDS ...\n- delete: timestamp for borg delete --info added, #4359\n\nOther changes:\n\n- redo stale lock handling, #3986\n drop BORG_HOSTNAME_IS_UNIQUE (please use BORG_HOST_ID if needed).\n borg now always assumes it has a unique host id - either automatically\n from fqdn plus uuid.getnode() or overridden via BORG_HOST_ID.\n- docs:\n\n - added Alpine Linux to distribution list\n - elaborate on append-only mode docs\n- vagrant:\n\n - darwin: new 10.12 box\n - freebsd: new 12.0 box\n - openbsd: new 6.4 box\n - misc. updates / fixes\n\n\nVersion 1.2.0a3 (2019-02-26)\n----------------------------\n\nFixes:\n\n- LibreSSL build fixes, #4403\n- dummy ACL/xattr code fixes (used by OpenBSD and others), #4403\n- create: fix openat/statat issues for root directory, #4405\n\n\nVersion 1.2.0a2 and earlier (2019-02-24)\n----------------------------------------\n\nNew features:\n\n- compact: \"borg compact\" needs to be used to free repository space by\n compacting the segments (reading sparse segments, rewriting still needed\n data to new segments, deleting the sparse segments).\n Borg < 1.2 invoked compaction automatically at the end of each repository\n writing command.\n Borg >= 1.2 does not do that any more to give better speed, more control,\n more segment file stability (== less stuff moving to newer segments) and\n more robustness.\n See the docs about \"borg compact\" for more details.\n- \"borg compact --cleanup-commits\" is to cleanup the tons of 17byte long\n commit-only segment files caused by borg 1.1.x issue #2850.\n Invoke this once after upgrading (the server side) borg to 1.2.\n Compaction now automatically removes unneeded commit-only segment files.\n- prune: Show which rule was applied to keep archive, #2886\n- add fixed blocksize chunker (see --chunker-params docs), #1086\n\nFixes:\n\n- avoid stale filehandle issues, #3265\n- use more FDs, avoid race conditions on active fs, #906, #908, #1038\n- add O_NOFOLLOW to base flags, #908\n- compact:\n\n - require >10% freeable space in a segment, #2985\n - repository compaction now automatically removes unneeded 17byte\n commit-only segments, #2850\n- make swidth available on all posix platforms, #2667\n\nOther changes:\n\n- repository: better speed and less stuff moving around by using separate\n segment files for manifest DELETEs and PUTs, #3947\n- use pyinstaller v3.3.1 to build binaries\n- update bundled zstd code to 1.3.8, #4210\n- update bundled lz4 code to 1.8.3, #4209\n- msgpack:\n\n - switch to recent \"msgpack\" pypi pkg name, #3890\n - wrap msgpack to avoid future compat complications, #3632, #2738\n - support msgpack 0.6.0 and 0.6.1, #4220, #4308\n\n- llfuse: modernize / simplify llfuse version requirements\n- code refactorings / internal improvements:\n\n - include size/csize/nfiles[_parts] stats into archive, #3241\n - calc_stats: use archive stats metadata, if available\n - crypto: refactored crypto to use an AEAD style API\n - crypto: new AES-OCB, CHACHA20-POLY1305\n - create: use less syscalls by not using a python file obj, #906, #3962\n - diff: refactor the diff functionality to new ItemDiff class, #2475\n - archive: create FilesystemObjectProcessors class\n - helpers: make a package, split into smaller modules\n - xattrs: move to platform package, use cython instead ctypes, #2495\n - xattrs/acls/bsdflags: misc. code/api optimizations\n - FUSE: separate creation of filesystem from implementation of llfuse funcs, #3042\n - FUSE: use unpacker.tell() instead of deprecated write_bytes, #3899\n - setup.py: move build_man / build_usage code to setup_docs.py\n - setup.py: update to use a newer Cython/setuptools API for compiling .pyx -> .c, #3788\n - use python 3.5's os.scandir / os.set_blocking\n - multithreading preparations (not used yet):\n\n - item.to_optr(), Item.from_optr()\n - fix chunker holding the GIL during blocking I/O\n - C code portability / basic MSC compatibility, #4147, #2677\n- testing:\n\n - vagrant: new VMs for linux/bsd/darwin, most with OpenSSL 1.1 and py36\n\n\nVersion 1.1.18 (2022-06-05)\n---------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux).\n If WSL still has a problem with sync_file_range, you need to set\n BORG_WORKAROUNDS=basesyncfile in the borg process environment to\n work around the WSL issue.\n- 1.1.14 changes return codes due to a bug fix:\n In case you have scripts expecting rc == 2 for a signal exit, you need to\n update them to check for >= 128 (as documented since long).\n- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now.\n- 1.1.17 install_requires the \"packaging\" pypi package now.\n\nNew features:\n\n- check --repair: significantly speed up search for next valid object in segment, #6022\n- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024\n- key export: display key if path is '-' or not given, #6092\n- list --format: add command_line to format keys, #6108\n\nFixes:\n\n- check: improve error handling for corrupt archive metadata block,\n make robust_iterator more robust, #4777\n- diff: support presence change for blkdev, chrdev and fifo items, #6483\n- diff: reduce memory consumption, fix is_hardlink_master\n- init: disallow overwriting of existing keyfiles\n- info: fix authenticated mode repo to show \"Encrypted: No\", #6462\n- info: emit repo info even if repo has 0 archives, #6120\n- list: remove placeholders for shake_* hashes, #6082\n- mount -o versions: give clear error msg instead of crashing\n- show_progress: add finished=true/false to archive_progress json, #6570\n- fix hardlinkable file type check, #6037\n- do not show archive name in error msgs referring to the repository, #6023\n- prettier error msg (no stacktrace) if exclude file is missing, #5734\n- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979\n- atomically create the CACHE_TAG file, #6028\n- deal with the SaveFile/SyncFile race, docs, see #6176 5c5b59bc9\n- avoid expanding path into LHS of formatting operation + tests, #6064 #6063\n- repository: quota / compactable computation fixes, #6119.\n This is mainly to keep the repo code in sync with borg 1.2. As borg 1.1\n compacts immediately, there was not really an issue with this in 1.1.\n- fix transaction rollback: use files cache filename as found in txn.active, #6353\n- do not load files cache for commands not using it, fixes #5673\n- fix scp repo url parsing for ip v6 addrs, #6526\n- repo::archive location placeholder expansion fixes, #5826, #5998\n\n - use expanded location for log output\n - support placeholder expansion for BORG_REPO env var\n- respect umask for created directory and file modes, #6400\n- safer truncate_and_unlink implementation\n\nOther changes:\n\n- upgrade bundled xxhash code to 0.8.1\n- fix xxh64 related build (setup.py and post-0.8.1 patch for static_assert).\n The patch was required to build the bundled xxhash code on FreeBSD, see\n https://github.com/Cyan4973/xxHash/pull/670\n- msgpack build: remove endianness macro, #6105\n- update and fix shell completions\n- fuse: remove unneeded version check and compat code\n- delete --force: do not ask when deleting a repo, #5941\n- delete: don't commit if nothing was deleted, avoid cache sync, #6060\n- delete: add repository id and location to prompt\n- compact segments: improve freeable / freed space log output, #5679\n- if ensure_dir() fails, give more informative error message, #5952\n- load_key: no key is same as empty key, #6441\n- better error msg for defect or unsupported repo configs, #6566\n- use hmac.compare_digest instead of ==, #6470\n- implement more standard hashindex.setdefault behaviour\n- remove stray punctuation from secure-erase message\n- add development.lock.txt, use a real python 3.5 to generate frozen reqs\n- setuptools 60.7.0 breaks pyinstaller, #6246\n- setup.py clean2 was added to work around some setuptools customizability limitation.\n- allow extra compiler flags for every extension build\n- C code: make switch fallthrough explicit\n- Cython code: fix \"useless trailing comma\" cython warnings\n- requirements.lock.txt: use the latest cython 0.29.30\n- fix compilation warnings: ‘PyUnicode_AsUnicode’ is deprecated\n- docs:\n\n - ~/.config/borg/keys is not used for repokey keys, #6107\n - excluded parent dir's metadata can't restore, #6062\n - permissions note rewritten to make it less confusing, #5490\n - add note about grandfather-father-son backup retention policy / rotation scheme\n - clarify who starts the remote agent (borg serve)\n - test/improve pull backup docs, #5903\n - document the socat pull mode described in #900 #515ß\n - borg serve: improve ssh forced commands docs, #6083\n - improve docs for borg list --format, #6080\n - fix the broken link to .nix file\n - clarify pattern usage with commands, #5176\n - clarify user_id vs uid for fuse, #5723\n - fix binary build freebsd/macOS version, #5942\n - FAQ: fix manifest-timestamp path, #6016\n - remove duplicate faq entries, #5926\n - fix sphinx warnings, #5919\n - virtualisation speed tips\n - fix values of TAG bytes, #6515\n - recommend umask for passphrase file perms\n - update link to ubuntu packages, #6485\n - clarify on-disk order and size of log entry fields, #6357\n - do not transform --/--- to unicode dashes\n - improve linking inside docs, link to borg_placeholders, link to borg_patterns\n - use same phrasing in misc. help texts\n - borg init: explain the encryption modes better\n - explain the difference between a path that ends with or without a slash, #6297\n - clarify usage of patternfile roots, #6242\n - borg key export: add examples\n - updates about features not experimental any more: FUSE \"versions\" view, --pattern*, #6134\n - fix/update cygwin package requirements\n - impact of deleting path/to/repo/nonce, #5858\n - warn about tampered server nonce\n - mention BORG_FILES_CACHE_SUFFIX as alternative to BORG_FILES_CACHE_TTL, #5602\n - add a troubleshooting note about \"is not a valid repository\" to the FAQ\n- vagrant / CI / testing:\n\n - misc. fixes and updates, new python versions\n - macOS on github: re-enable fuse2 testing by downgrading to older macOS, #6099\n - fix OpenBSD symlink mode test failure, #2055\n - use the generic/openbsd6 box\n - strengthen the test: we can read data w/o nonces\n - add tests for path/to/repo/nonce deletion\n - darwin64: backport some tunings from master\n - darwin64: remove fakeroot, #6314\n - darwin64: fix vagrant scp, #5921\n - darwin64: use macfuse instead of osxfuse\n - add ubuntu \"jammy\" 22.04 LTS VM\n - adapt memory for openindiana64 and darwin64\n\n\nVersion 1.1.17 (2021-07-12)\n---------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux).\n If WSL still has a problem with sync_file_range, you need to set\n BORG_WORKAROUNDS=basesyncfile in the borg process environment to\n work around the WSL issue.\n- 1.1.14 changes return codes due to a bug fix:\n In case you have scripts expecting rc == 2 for a signal exit, you need to\n update them to check for >= 128 (as documented since long).\n- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now.\n- 1.1.17 install_requires the \"packaging\" pypi package now.\n\nFixes:\n\n- pyinstaller dir-mode: fix pyi detection / LIBPATH treatment, #5897\n- handle crash due to kill stale lock race, #5828\n- fix BORG_CACHE_DIR crashing borg if empty, #5216\n- create --dry-run: fix display of kept tagfile, #5834\n- fix missing parameter in \"did not consistently fail\" msg, #5822\n- missing / healed chunks: always tell chunk ID, #5704\n- benchmark: make sure cleanup happens even on exceptions, #5630\n\nNew features:\n\n- implement BORG_SELFTEST env variable, #5871.\n this can be used to accelerate borg startup a bit. not recommended for\n normal usage, but borg mass hosters with a lot of borg invocations can\n save some resources with this. on my laptop, this saved ~100ms cpu time\n (sys+user) per borg command invocation.\n- implement BORG_LIBC env variable to give the libc filename, #5870.\n you can use this if a borg does not find your libc.\n- check: add progress indicator for archive check.\n- allow --files-cache=size (not recommended, make sure you know what you do)\n\nOther changes:\n\n- Python 3.10 now officially supported!\n we test on py310-dev on github CI since a while and now also on the vagrant\n machines, so it should work ok.\n- github CI: test on py310 (again)\n- get rid of distutils, use packaging and setuptools.\n distutils is deprecated and gives warnings on py 3.10.\n- setup.py: rename \"clean\" to \"clean2\" to avoid shadowing the \"clean\" command.\n- remove libc filename fallback for the BSDs (there is no \"usual\" name)\n- cleanup flake8 checks, fix some pep8 violations.\n- docs building: replace deprecated function \".add_stylesheet()\" for Sphinx 4 compatibility\n- docs:\n\n - add a hint on sleeping computer and ssh connections, #5301\n - update the documentation on hacked backup client, #5480\n - improve docs/FAQ about append-only remote repos, #5497\n - complement the documentation for pattern files and exclude files, #5520\n - \"filename with spaces\" example added to exclude file, #5236\n note: no whitespace escaping needed, processed by borg.\n - add info on renaming repositories, #5240\n - clarify borg check --verify-data, #5808\n - add notice about defective hardware to check documentation, #5753\n - add paragraph added in #5855 to utility documentation source\n - add missing leading slashes in help patterns, #5857\n - clarify \"you will need key and passphrase\" borg init warning, #4622\n - pull mode: add some warnings, #5827\n - mention tar --compare (compare archive to fs files), #5880\n - fix typos, backport of #5597\n- vagrant:\n\n - add py3.7.11 for binary build, also add 3.10-dev.\n - use latest Cython 0.29.23 for py310 compat fixes.\n - more RAM for openindiana upgrade plan resolver, it just hangs (swaps?) if\n there is too little RAM.\n - fix install_pyenv to adapt to recent changes in pyenv (same as in master now).\n - use generic/netbsd9 box, copied from master branch.\n\n\nVersion 1.1.16 (2021-03-23)\n---------------------------\n\nFixes:\n\n- setup.py: add special openssl prefix for Apple M1 compatibility\n- do not recurse into duplicate roots, #5603\n- remove empty shadowed_segments lists, #5275, #5614\n- fix libpython load error when borg fat binary / dir-based binary is invoked\n via a symlink by upgrading pyinstaller to v4.2, #5688\n- config: accept non-int value (like 500M or 100G) for max_segment_size or\n storage_quota, #5639.\n please note: when setting a non-int value for this in a repo config,\n using the repo will require borg >= 1.1.16.\n\nNew features:\n\n- bundled msgpack: drop support for old buffer protocol to support Python 3.10\n- verbose files cache logging via --debug-topic=files_cache, #5659.\n Use this if you suspect that borg does not detect unmodified files as expected.\n- create/extract: add --noxattrs and --noacls option, #3955.\n when given with borg create, borg will not get xattrs / ACLs from input files\n (and thus, it will not archive xattrs / ACLs). when given with borg extract,\n borg will not read xattrs / ACLs from archive and will not set xattrs / ACLs\n on extracted files.\n- diff: add --json-lines option, #3765\n- check: debug log segment filename\n- borg debug dump-hints\n\nOther changes:\n\n- Tab completion support for additional archives for 'borg delete'\n- repository: deduplicate code of put and delete, no functional change\n- tests: fix result order issue (sporadic test failure on openindiana)\n- vagrant:\n\n - upgrade pyinstaller to v4.2, #5671\n - avoid grub-install asking interactively for device\n - remove the xenial box\n - update freebsd box to 12.1\n- docs:\n\n - update macOS install instructions, #5677\n - use macFUSE (not osxfuse) for Apple M1 compatibility\n - update docs for dev environment installation instructions, #5643\n - fix grammar in faq\n - recommend running tests only on installed versions for setup\n - add link back to git-installation\n - remove /var/cache exclusion in example commands, #5625.\n This is generally a poor idea and shouldn't be promoted through examples.\n - add repology.org badge with current packaging status\n - explain hash collision\n - add unsafe workaround to use an old repo copy, #5722\n\n\nVersion 1.1.15 (2020-12-25)\n---------------------------\n\nFixes:\n\n- extract:\n\n - improve exception handling when setting xattrs, #5092.\n - emit a warning message giving the path, xattr key and error message.\n - continue trying to restore other xattrs and bsdflags of the same file\n after an exception with xattr-setting happened.\n- export-tar:\n\n - set tar format to GNU_FORMAT explicitly, #5274\n - fix memory leak with ssh: remote repository, #5568\n - fix potential memory leak with ssh: remote repository with partial extraction\n- create: fix --dry-run and --stats coexistence, #5415\n- use --timestamp for {utcnow} and {now} if given, #5189\n\nNew features:\n\n- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333\n- allow appending the files cache filename with BORG_FILES_CACHE_SUFFIX env var\n\nOther changes:\n\n- drop python 3.4 support, minimum requirement is 3.5 now.\n- enable using libxxhash instead of bundled xxh64 code\n- update llfuse requirements (1.3.8)\n- set cython language_level in some files to fix warnings\n- allow EIO with warning when trying to hardlink\n- PropDict: fail early if internal_dict is not a dict\n- update shell completions\n- tests / CI\n\n - add a test for the hashindex corruption bug, #5531 #4829\n - fix spurious failure in test_cache_files, #5438\n - added a github ci workflow\n - reduce testing on travis, no macOS, no py3x-dev, #5467\n - travis: use newer dists, native py on dist\n- vagrant:\n\n - remove jessie and trusty boxes, #5348 #5383\n - pyinstaller 4.0, build on py379\n - binary build on stretch64, #5348\n - remove easy_install based pip installation\n- docs:\n\n - clarify '--one-file-system' for btrfs, #5391\n - add example for excluding content using the --pattern cmd line arg\n - complement the documentation for pattern files and exclude files, #5524\n - made ansible playbook more generic, use package instead of pacman. also\n change state from \"latest\" to \"present\".\n - complete documentation on append-only remote repos, #5497\n - internals: rather talk about target size than statistics, #5336\n - new compression algorithm policy, #1633 #5505\n - faq: add a hint on sleeping computer, #5301\n - note requirements for full disk access on macOS Catalina, #5303\n - fix/improve description of borg upgrade hardlink usage, #5518\n- modernize 1.1 code:\n\n - drop code/workarounds only needed to support Python 3.4\n - remove workaround for pre-release py37 argparse bug\n - removed some outdated comments/docstrings\n - requirements: remove some restrictions, lock on current versions\n\n\nVersion 1.1.14 (2020-10-07)\n---------------------------\n\nFixes:\n\n- check --repair: fix potential data loss when interrupting it, #5325\n- exit with 128 + signal number (as documented) when borg is killed by a signal, #5161\n- fix hardlinked CACHEDIR.TAG processing, #4911\n- create --read-special: .part files also should be regular files, #5217\n- llfuse dependency: choose least broken 1.3.6/1.3.7.\n 1.3.6 is broken on python 3.9, 1.3.7 is broken on FreeBSD.\n\nOther changes:\n\n- upgrade bundled xxhash to 0.7.4\n- self test: if it fails, also point to OS and hardware, #5334\n- pyinstaller: compute basepath from spec file location\n- prettier error message when archive gets too big, #5307\n- check/recreate are not \"experimental\" any more (but still potentially dangerous):\n\n - recreate: remove extra confirmation\n - rephrase some warnings, update docs, #5164\n- shell completions:\n\n - misc. updates / fixes\n - support repositories in fish tab completion, #5256\n - complete $BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING\n - rewrite zsh completion:\n\n - completion for almost all optional and positional arguments\n - completion for Borg environment variables (parameters)\n- use \"allow/deny list\" instead of \"white/black list\" wording\n- declare \"allow_cache_wipe\" marker in setup.cfg to avoid pytest warning\n- vagrant / tests:\n\n - misc. fixes / updates\n - use python 3.5.10 for binary build\n - build directory-based binaries additionally to the single file binaries\n - add libffi-dev, required to build python\n - use cryptography<3.0, more recent versions break the jessie box\n - test on python 3.9\n - do brew update with /dev/null redirect to avoid \"too much log output\" on travis-ci\n- docs:\n\n - add ssh-agent pull backup method docs, #5288\n - how to approach borg speed issues, #5371\n - mention double --force in prune docs\n - update Homebrew install instructions, #5185\n - better description of how cache and rebuilds of it work\n - point to borg create --list item flags in recreate usage, #5165\n - add security faq explaining AES-CTR crypto issues, #5254\n - add a note to create from stdin regarding files cache, #5180\n - fix borg.1 manpage generation regression, #5211\n - clarify how exclude options work in recreate, #5193\n - add section for retired contributors\n - hint about not misusing private email addresses of contributors for borg support\n\n\nVersion 1.1.13 (2020-06-06)\n---------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux).\n If WSL still has a problem with sync_file_range, you need to set\n BORG_WORKAROUNDS=basesyncfile in the borg process environment to\n work around the WSL issue.\n\nFixes:\n\n- rebuilt using a current Cython version, compatible with python 3.8, #5214\n\n\nVersion 1.1.12 (2020-06-06)\n---------------------------\n\nFixes:\n\n- fix preload-related memory leak, #5202.\n- mount / borgfs (FUSE filesystem):\n\n - fix FUSE low linear read speed on large files, #5067\n - fix crash on old llfuse without birthtime attrs, #5064 - accidentally\n we required llfuse >= 1.3. Now also old llfuse works again.\n - set f_namemax in statfs result, #2684\n- update precedence of env vars to set config and cache paths, #4894\n- correctly calculate compression ratio, taking header size into account, too\n\nNew features:\n\n- --bypass-lock option to bypass locking with read-only repositories\n\nOther changes:\n\n- upgrade bundled zstd to 1.4.5\n- travis: adding comments and explanations to Travis config / install script,\n improve macOS builds.\n- tests: test_delete_force: avoid sporadic test setup issues, #5196\n- misc. vagrant fixes\n- the binary for macOS is now built on macOS 10.12\n- the binaries for Linux are now built on Debian 8 \"Jessie\", #3761\n- docs:\n\n - PlaceholderError not printed as JSON, #4073\n - \"How important is Borg config?\", #4941\n - make Sphinx warnings break docs build, #4587\n - some markup / warning fixes\n - add \"updating borgbackup.org/releases\" to release checklist, #4999\n - add \"rendering docs\" to release checklist, #5000\n - clarify borg init's encryption modes\n - add note about patterns and stored paths, #4160\n - add upgrade of tools to pip installation how-to\n - document one cause of orphaned chunks in check command, #2295\n - linked recommended restrictions to ssh public keys on borg servers in faq, #4946\n\n\nVersion 1.1.11 (2020-03-08)\n---------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux).\n If WSL still has a problem with sync_file_range, you need to set\n BORG_WORKAROUNDS=basesyncfile in the borg process environment to\n work around the WSL issue.\n\nFixes:\n\n- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829.\n Please read and follow the more detailed notes close to the top of this document.\n- upgrade bundled xxhash to 0.7.3, #4891.\n 0.7.2 is the minimum requirement for correct operations on ARMv6 in non-fixup\n mode, where unaligned memory accesses cause bus errors.\n 0.7.3 adds some speedups and libxxhash 0.7.3 even has a pkg-config file now.\n- upgrade bundled lz4 to 1.9.2\n- upgrade bundled zstd to 1.4.4\n- fix crash when upgrading erroneous hints file, #4922\n- extract:\n\n - fix KeyError for \"partial\" extraction, #4607\n - fix \"partial\" extract for hardlinked contentless file types, #4725\n - fix preloading for old (0.xx) remote servers, #4652\n - fix confusing output of borg extract --list --strip-components, #4934\n- delete: after double-force delete, warn about necessary repair, #4704\n- create: give invalid repo error msg if repo config not found, #4411\n- mount: fix FUSE mount missing st_birthtime, #4763 #4767\n- check: do not stumble over invalid item key, #4845\n- info: if the archive doesn't exist, print a pretty message, #4793\n- SecurityManager.known(): check all files, #4614\n- Repository.open: use stat() to check for repo dir, #4695\n- Repository.check_can_create_repository: use stat() to check, #4695\n- fix invalid archive error message\n- fix optional/non-optional location arg, #4541\n- commit-time free space calc: ignore bad compact map entries, #4796\n- ignore EACCES (errno 13) when hardlinking the old config, #4730\n- --prefix / -P: fix processing, avoid argparse issue, #4769\n\nNew features:\n\n- enable placeholder usage in all extra archive arguments\n- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710\n- recreate: support --timestamp option, #4745\n- support platforms without os.link (e.g. Android with Termux), #4901.\n if we don't have os.link, we just extract another copy instead of making a hardlink.\n- support linux platforms without sync_file_range (e.g. Android 7 with Termux), #4905\n\nOther:\n\n- ignore --stats when given with --dry-run, but continue, #4373\n- add some ProgressIndicator msgids to code / fix docs, #4935\n- elaborate on \"Calculating size\" message\n- argparser: always use REPOSITORY in metavar, also use more consistent help phrasing.\n- check: improve error output for matching index size, see #4829\n- docs:\n\n - changelog: add advisory about hashindex_set bug #4829\n - better describe BORG_SECURITY_DIR, BORG_CACHE_DIR, #4919\n - infos about cache security assumptions, #4900\n - add FAQ describing difference between a local repo vs. repo on a server.\n - document how to test exclusion patterns without performing an actual backup\n - timestamps in the files cache are now usually ctime, #4583\n - fix bad reference to borg compact (does not exist in 1.1), #4660\n - create: borg 1.1 is not future any more\n - extract: document limitation \"needs empty destination\", #4598\n - how to supply a passphrase, use crypto devices, #4549\n - fix osxfuse github link in installation docs\n - add example of exclude-norecurse rule in help patterns\n - update macOS Brew link\n - add note about software for automating backups, #4581\n - AUTHORS: mention copyright+license for bundled msgpack\n - fix various code blocks in the docs, #4708\n - updated docs to cover use of temp directory on remote, #4545\n - add restore docs, #4670\n - add a pull backup / push restore how-to, #1552\n - add FAQ how to retain original paths, #4532\n - explain difference between --exclude and --pattern, #4118\n - add FAQs for SSH connection issues, #3866\n - improve password FAQ, #4591\n - reiterate that 'file cache names are absolute' in FAQ\n- tests:\n\n - cope with ANY error when importing pytest into borg.testsuite, #4652\n - fix broken test that relied on improper zlib assumptions\n - test_fuse: filter out selinux xattrs, #4574\n- travis / vagrant:\n\n - misc python versions removed / changed (due to openssl 1.1 compatibility)\n or added (3.7 and 3.8, for better borg compatibility testing)\n - binary building is on python 3.5.9 now\n- vagrant:\n\n - add new boxes: ubuntu 18.04 and 20.04, debian 10\n - update boxes: openindiana, darwin, netbsd\n - remove old boxes: centos 6\n - darwin: updated osxfuse to 3.10.4\n - use debian/ubuntu pip/virtualenv packages\n - rather use python 3.6.2 than 3.6.0, fixes coverage/sqlite3 issue\n - use requirements.d/development.lock.txt to avoid compat issues\n- travis:\n\n - darwin: backport some install code / order from master\n - remove deprecated keyword \"sudo\" from travis config\n - allow osx builds to fail, #4955\n this is due to travis-ci frequently being so slow that the OS X builds\n just fail because they exceed 50 minutes and get killed by travis.\n\n\nVersion 1.1.10 (2019-05-16)\n---------------------------\n\nFixes:\n\n- extract: hang on partial extraction with ssh: repo, when hardlink master\n is not matched/extracted and borg hangs on related slave hardlink, #4350\n- lrucache: regularly remove old FDs, #4427\n- avoid stale filehandle issues, #3265\n- freebsd: make xattr platform code api compatible with linux, #3952\n- use whitelist approach for borg serve, #4097\n- borg command shall terminate with rc 2 for ImportErrors, #4424\n- create: only run stat_simple_attrs() once, this increases\n backup with lots of unchanged files performance by ~ 5%.\n- prune: fix incorrect borg prune --stats output with --dry-run, #4373\n- key export: emit user-friendly error if repo key is exported to a directory,\n #4348\n\nNew features:\n\n- bundle latest supported msgpack-python release (0.5.6), remove msgpack-python\n from setup.py install_requires - by default we use the bundled code now.\n optionally, we still support using an external msgpack (see hints in\n setup.py), but this requires solid requirements management within\n distributions and is not recommended.\n borgbackup will break if you upgrade msgpack to an unsupported version.\n- display msgpack version as part of sysinfo (e.g. in tracebacks)\n- timestamp for borg delete --info added, #4359\n- enable placeholder usage in --comment and --glob-archives, #4559, #4495\n\nOther:\n\n- serve: do not check python/libc for borg serve, #4483\n- shell completions: borg diff second archive\n- release scripts: signing binaries with Qubes OS support\n- testing:\n\n - vagrant: upgrade openbsd box to 6.4\n - travis-ci: lock test env to py 3.4 compatible versions, #4343\n - get rid of confusing coverage warning, #2069\n - rename test_mount_hardlinks to test_fuse_mount_hardlinks,\n so both can be excluded by \"not test_fuse\".\n - pure-py msgpack warning shall not make a lot of tests fail, #4558\n- docs:\n\n - add \"SSH Configuration\" section to \"borg serve\", #3988, #636, #4485\n - README: new URL for funding options\n - add a sample logging.conf in docs/misc, #4380\n - elaborate on append-only mode docs, #3504\n - installation: added Alpine Linux to distribution list, #4415\n - usage.html: only modify window.location when redirecting, #4133\n - add msgpack license to docs/3rd_party/msgpack\n- vagrant / binary builds:\n\n - use python 3.5.7 for builds\n - use osxfuse 3.8.3\n\n\nVersion 1.1.9 (2019-02-10)\n--------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n\nFixes:\n\n- security fix: configure FUSE with \"default_permissions\", #3903\n \"default_permissions\" is now enforced by borg by default to let the\n kernel check uid/gid/mode based permissions.\n \"ignore_permissions\" can be given not to enforce \"default_permissions\".\n- make \"hostname\" short, even on misconfigured systems, #4262\n- fix free space calculation on macOS (and others?), #4289\n- config: quit with error message when no key is provided, #4223\n- recover_segment: handle too small segment files correctly, #4272\n- correctly release memoryview, #4243\n- avoid diaper pattern in configparser by opening files, #4263\n- add \"# cython: language_level=3\" directive to .pyx files, #4214\n- info: consider part files for \"This archive\" stats, #3522\n- work around Microsoft WSL issue #645 (sync_file_range), #1961\n\nNew features:\n\n- add --rsh command line option to complement BORG_RSH env var, #1701\n- init: --make-parent-dirs parent1/parent2/repo_dir, #4235\n\nOther:\n\n- add archive name to check --repair output, #3447\n- check for unsupported msgpack versions\n- shell completions:\n\n - new shell completions for borg 1.1.9\n - more complete shell completions for borg mount -o\n - added shell completions for borg help\n - option arguments for zsh tab completion\n- docs:\n\n - add FAQ regarding free disk space check, #3905\n - update BORG_PASSCOMMAND example and clarify variable expansion, #4249\n - FAQ regarding change of compression settings, #4222\n - add note about BSD flags to changelog, #4246\n - improve logging in example automation script\n - add note about files changing during backup, #4081\n - work around the backslash issue, #4280\n - update release workflow using twine (docs, scripts), #4213\n - add warnings on repository copies to avoid future problems, #4272\n- tests:\n\n - fix the homebrew 1.9 issues on travis-ci, #4254\n - fix duplicate test method name, #4311\n\n\nVersion 1.1.8 (2018-12-09)\n--------------------------\n\nFixes:\n\n- enforce storage quota if set by serve-command, #4093\n- invalid locations: give err msg containing parsed location, #4179\n- list repo: add placeholders for hostname and username, #4130\n- on linux, symlinks can't have ACLs, so don't try to set any, #4044\n\nNew features:\n\n- create: added PATH::archive output on INFO log level\n- read a passphrase from a file descriptor specified in the\n BORG_PASSPHRASE_FD environment variable.\n\nOther:\n\n- docs:\n\n - option --format is required for some expensive-to-compute values for json\n\n borg list by default does not compute expensive values except when\n they are needed. whether they are needed is determined by the format,\n in standard mode as well as in --json mode.\n - tell that our binaries are x86/x64 amd/intel, bauerj has ARM\n - fixed wrong archive name pattern in CRUD benchmark help\n - fixed link to cachedir spec in docs, #4140\n- tests:\n\n - stop using fakeroot on travis, avoids sporadic EISDIR errors, #2482\n - xattr key names must start with \"user.\" on linux\n - fix code so flake8 3.6 does not complain\n - explicitly convert environment variable to str, #4136\n - fix DeprecationWarning: Flags not at the start of the expression, #4137\n - support pytest4, #4172\n- vagrant:\n\n - use python 3.5.6 for builds\n\n\nVersion 1.1.7 (2018-08-11)\n--------------------------\n\nCompatibility notes:\n\n- added support for Python 3.7\n\nFixes:\n\n- cache lock: use lock_wait everywhere to fix infinite wait, see #3968\n- don't archive tagged dir when recursing an excluded dir, #3991\n- py37 argparse: work around bad default in py 3.7.0a/b/rc, #3996\n- py37 remove loggerDict.clear() from tearDown method, #3805\n- some fixes for bugs which likely did not result in problems in practice:\n\n - fixed logic bug in platform module API version check\n - fixed xattr/acl function prototypes, added missing ones\n\nNew features:\n\n- init: add warning to store both key and passphrase at safe place(s)\n- BORG_HOST_ID env var to work around all-zero MAC address issue, #3985\n- borg debug dump-repo-objs --ghost (dump everything from segment files,\n including deleted or superseded objects or commit tags)\n- borg debug search-repo-objs (search in repo objects for hex bytes or strings)\n\nOther changes:\n\n- add Python 3.7 support\n- updated shell completions\n- call socket.gethostname only once\n- locking: better logging, add some asserts\n- borg debug dump-repo-objs:\n\n - filename layout improvements\n - use repository.scan() to get on-disk order\n- docs:\n\n - update installation instructions for macOS\n - added instructions to install fuse via homebrew\n - improve diff docs\n - added note that checkpoints inside files requires 1.1+\n - add link to tempfile module\n - remove row/column-spanning from docs source, #4000 #3990\n- tests:\n\n - fetch less data via os.urandom\n - add py37 env for tox\n - travis: add 3.7, remove 3.6-dev (we test with -dev in master)\n- vagrant / binary builds:\n\n - use osxfuse 3.8.2\n - use own (uptodate) openindiana box\n\n\nVersion 1.1.6 (2018-06-11)\n--------------------------\n\nCompatibility notes:\n\n- 1.1.6 changes:\n\n - also allow msgpack-python 0.5.6.\n\nFixes:\n\n- fix borg exception handling on ENOSPC error with xattrs, #3808\n- prune: fix/improve overall progress display\n- borg config repo ... does not need cache/manifest/key, #3802\n- debug dump-repo-objs should not depend on a manifest obj\n- pypi package:\n\n - include .coveragerc, needed by tox.ini\n - fix package long description, #3854\n\nNew features:\n\n- mount: add uid, gid, umask mount options\n- delete:\n\n - only commit once, #3823\n - implement --dry-run, #3822\n- check:\n\n - show progress while rebuilding missing manifest, #3787\n - more --repair output\n- borg config --list , #3612\n\nOther changes:\n\n- update msgpack requirement, #3753\n- update bundled zstd to 1.3.4, #3745\n- update bundled lz4 code to 1.8.2, #3870\n- docs:\n\n - describe what BORG_LIBZSTD_PREFIX does\n - fix and deduplicate encryption quickstart docs, #3776\n- vagrant:\n\n - FUSE for macOS: upgrade 3.7.1 to 3.8.0\n - exclude macOS High Sierra upgrade on the darwin64 machine\n - remove borgbackup.egg-info dir in fs_init (after rsync)\n - use pyenv-based build/test on jessie32/62\n - use local 32 and 64bit debian jessie boxes\n - use \"vagrant\" as username for new xenial box\n- travis OS X: use xcode 8.3 (not broken)\n\n\nVersion 1.1.5 (2018-04-01)\n--------------------------\n\nCompatibility notes:\n\n- 1.1.5 changes:\n\n - require msgpack-python >= 0.4.6 and < 0.5.0.\n 0.5.0+ dropped python 3.4 testing and also caused some other issues because\n the python package was renamed to msgpack and emitted some FutureWarning.\n\nFixes:\n\n- create --list: fix that it was never showing M status, #3492\n- create: fix timing for first checkpoint (read files cache early, init\n checkpoint timer after that), see #3394\n- extract: set rc=1 when extracting damaged files with all-zero replacement\n chunks or with size inconsistencies, #3448\n- diff: consider an empty file as different to a non-existing file, #3688\n- files cache: improve exception handling, #3553\n- ignore exceptions in scandir_inorder() caused by an implicit stat(),\n also remove unneeded sort, #3545\n- fixed tab completion problem where a space is always added after path even\n when it shouldn't\n- build: do .h file content checks in binary mode, fixes build issue for\n non-ascii header files on pure-ascii locale platforms, #3544 #3639\n- borgfs: fix patterns/paths processing, #3551\n- config: add some validation, #3566\n- repository config: add validation for max_segment_size, #3592\n- set cache previous_location on load instead of save\n- remove platform.uname() call which caused library mismatch issues, #3732\n- add exception handler around deprecated platform.linux_distribution() call\n- use same datetime object for {now} and {utcnow}, #3548\n\nNew features:\n\n- create: implement --stdin-name, #3533\n- add chunker_params to borg archive info (--json)\n- BORG_SHOW_SYSINFO=no to hide system information from exceptions\n\nOther changes:\n\n- updated zsh completions for borg 1.1.4\n- files cache related code cleanups\n- be more helpful when parsing invalid --pattern values, #3575\n- be more clear in secure-erase warning message, #3591\n- improve getpass user experience, #3689\n- docs build: unicode problem fixed when using a py27-based sphinx\n- docs:\n\n - security: explicitly note what happens OUTSIDE the attack model\n - security: add note about combining compression and encryption\n - security: describe chunk size / proximity issue, #3687\n - quickstart: add note about permissions, borg@localhost, #3452\n - quickstart: add introduction to repositories & archives, #3620\n - recreate --recompress: add missing metavar, clarify description, #3617\n - improve logging docs, #3549\n - add an example for --pattern usage, #3661\n - clarify path semantics when matching, #3598\n - link to offline documentation from README, #3502\n - add docs on how to verify a signed release with GPG, #3634\n - chunk seed is generated per repository (not: archive)\n - better formatting of CPU usage documentation, #3554\n - extend append-only repo rollback docs, #3579\n- tests:\n\n - fix erroneously skipped zstd compressor tests, #3606\n - skip a test if argparse is broken, #3705\n- vagrant:\n\n - xenial64 box now uses username 'vagrant', #3707\n - move cleanup steps to fs_init, #3706\n - the boxcutter wheezy boxes are 404, use local ones\n - update to Python 3.5.5 (for binary builds)\n\n\nVersion 1.1.4 (2017-12-31)\n--------------------------\n\nCompatibility notes:\n\n- When upgrading from borg 1.0.x to 1.1.x, please note:\n\n - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.\n - borg upgrade: you do not need to and you also should not run it.\n - borg might ask some security-related questions once after upgrading.\n You can answer them either manually or via environment variable.\n One known case is if you use unencrypted repositories, then it will ask\n about a unknown unencrypted repository one time.\n - your first backup with 1.1.x might be significantly slower (it might\n completely read, chunk, hash a lot files) - this is due to the\n --files-cache mode change (and happens every time you change mode).\n You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible\n mode (but that is less safe for detecting changed files than the default).\n See the --files-cache docs for details.\n- borg 1.1.4 changes:\n\n - zstd compression is new in borg 1.1.4, older borg can't handle it.\n - new minimum requirements for the compression libraries - if the required\n versions (header and lib) can't be found at build time, bundled code will\n be used:\n\n - added requirement: libzstd >= 1.3.0 (bundled: 1.3.2)\n - updated requirement: liblz4 >= 1.7.0 / r129 (bundled: 1.8.0)\n\nFixes:\n\n- check: data corruption fix: fix for borg check --repair malfunction, #3444.\n See the more detailed notes close to the top of this document.\n- delete: also delete security dir when deleting a repo, #3427\n- prune: fix building the \"borg prune\" man page, #3398\n- init: use given --storage-quota for local repo, #3470\n- init: properly quote repo path in output\n- fix startup delay with dns-only own fqdn resolving, #3471\n\nNew features:\n\n- added zstd compression. try it!\n- added placeholder {reverse-fqdn} for fqdn in reverse notation\n- added BORG_BASE_DIR environment variable, #3338\n\nOther changes:\n\n- list help topics when invalid topic is requested\n- fix lz4 deprecation warning, requires lz4 >= 1.7.0 (r129)\n- add parens for C preprocessor macro argument usages (did not cause malfunction)\n- exclude broken pytest 3.3.0 release\n- updated fish/bash completions\n- init: more clear exception messages for borg create, #3465\n- docs:\n\n - add auto-generated docs for borg config\n - don't generate HTML docs page for borgfs, #3404\n - docs update for lz4 b2 zstd changes\n - add zstd to compression help, readme, docs\n - update requirements and install docs about bundled lz4 and zstd\n- refactored build of the compress and crypto.low_level extensions, #3415:\n\n - move some lib/build related code to setup_{zstd,lz4,b2}.py\n - bundle lz4 1.8.0 (requirement: >= 1.7.0 / r129)\n - bundle zstd 1.3.2 (requirement: >= 1.3.0)\n - blake2 was already bundled\n - rename BORG_LZ4_PREFIX env var to BORG_LIBLZ4_PREFIX for better consistency:\n we also have BORG_LIBB2_PREFIX and BORG_LIBZSTD_PREFIX now.\n - add prefer_system_lib* = True settings to setup.py - by default the build\n will prefer a shared library over the bundled code, if library and headers\n can be found and meet the minimum requirements.\n\n\nVersion 1.1.3 (2017-11-27)\n--------------------------\n\nFixes:\n\n- Security Fix for CVE-2017-15914: Incorrect implementation of access controls\n allows remote users to override repository restrictions in Borg servers.\n A user able to access a remote Borg SSH server is able to circumvent access\n controls post-authentication.\n Affected releases: 1.1.0, 1.1.1, 1.1.2. Releases 1.0.x are NOT affected.\n- crc32: deal with unaligned buffer, add tests - this broke borg on older ARM\n CPUs that can not deal with unaligned 32bit memory accesses and raise a bus\n error in such cases. the fix might also improve performance on some CPUs as\n all 32bit memory accesses by the crc32 code are properly aligned now. #3317\n- mount: fixed support of --consider-part-files and do not show .borg_part_N\n files by default in the mounted FUSE filesystem. #3347\n- fixed cache/repo timestamp inconsistency message, highlight that information\n is obtained from security dir (deleting the cache will not bypass this error\n in case the user knows this is a legitimate repo).\n- borgfs: don't show sub-command in borgfs help, #3287\n- create: show an error when --dry-run and --stats are used together, #3298\n\nNew features:\n\n- mount: added exclusion group options and paths, #2138\n\n Reused some code to support similar options/paths as borg extract offers -\n making good use of these to mount only a smaller subset of dirs/files can\n speed up mounting a lot and also will consume way less memory.\n\n borg mount [options] repo_or_archive mountpoint path [paths...]\n\n paths: you can just give some \"root paths\" (like for borg extract) to\n only partially populate the FUSE filesystem.\n\n new options: --exclude[-from], --pattern[s-from], --strip-components\n- create/extract: support st_birthtime on platforms supporting it, #3272\n- add \"borg config\" command for querying/setting/deleting config values, #3304\n\nOther changes:\n\n- clean up and simplify packaging (only package committed files, do not install\n .c/.h/.pyx files)\n- docs:\n\n - point out tuning options for borg create, #3239\n - add instructions for using ntfsclone, zerofree, #81\n - move image backup-related FAQ entries to a new page\n - clarify key aliases for borg list --format, #3111\n - mention break-lock in checkpointing FAQ entry, #3328\n - document sshfs rename workaround, #3315\n - add FAQ about removing files from existing archives\n - add FAQ about different prune policies\n - usage and man page for borgfs, #3216\n - clarify create --stats duration vs. wall time, #3301\n - clarify encrypted key format for borg key export, #3296\n - update release checklist about security fixes\n - document good and problematic option placements, fix examples, #3356\n - add note about using --nobsdflags to avoid speed penalty related to\n bsdflags, #3239\n - move most of support section to www.borgbackup.org\n\n\nVersion 1.1.2 (2017-11-05)\n--------------------------\n\nFixes:\n\n- fix KeyError crash when talking to borg server < 1.0.7, #3244\n- extract: set bsdflags last (include immutable flag), #3263\n- create: don't do stat() call on excluded-norecurse directory, fix exception\n handling for stat() call, #3209\n- create --stats: do not count data volume twice when checkpointing, #3224\n- recreate: move chunks_healthy when excluding hardlink master, #3228\n- recreate: get rid of chunks_healthy when rechunking (does not match), #3218\n- check: get rid of already existing not matching chunks_healthy metadata, #3218\n- list: fix stdout broken pipe handling, #3245\n- list/diff: remove tag-file options (not used), #3226\n\nNew features:\n\n- bash, zsh and fish shell auto-completions, see scripts/shell_completions/\n- added BORG_CONFIG_DIR env var, #3083\n\nOther changes:\n\n- docs:\n\n - clarify using a blank passphrase in keyfile mode\n - mention \"!\" (exclude-norecurse) type in \"patterns\" help\n - document to first heal before running borg recreate to re-chunk stuff,\n because that will have to get rid of chunks_healthy metadata.\n - more than 23 is not supported for CHUNK_MAX_EXP, #3115\n - borg does not respect nodump flag by default any more\n - clarify same-filesystem requirement for borg upgrade, #2083\n - update / rephrase cygwin / WSL status, #3174\n - improve docs about --stats, #3260\n- vagrant: openindiana new clang package\n\nAlready contained in 1.1.1 (last minute fix):\n\n- arg parsing: fix fallback function, refactor, #3205. This is a fixup\n for #3155, which was broken on at least python <= 3.4.2.\n\n\nVersion 1.1.1 (2017-10-22)\n--------------------------\n\nCompatibility notes:\n\n- The deprecated --no-files-cache is not a global/common option any more,\n but only available for borg create (it is not needed for anything else).\n Use --files-cache=disabled instead of --no-files-cache.\n- The nodump flag (\"do not back up this file\") is not honoured any more by\n default because this functionality (esp. if it happened by error or\n unexpected) was rather confusing and unexplainable at first to users.\n If you want that \"do not back up NODUMP-flagged files\" behaviour, use:\n borg create --exclude-nodump ...\n- If you are on Linux and do not need bsdflags archived, consider using\n ``--nobsdflags`` with ``borg create`` to avoid additional syscalls and\n speed up backup creation.\n\nFixes:\n\n- borg recreate: correctly compute part file sizes. fixes cosmetic, but\n annoying issue as borg check complains about size inconsistencies of part\n files in affected archives. you can solve that by running borg recreate on\n these archives, see also #3157.\n- bsdflags support: do not open BLK/CHR/LNK files, avoid crashes and\n slowness, #3130\n- recreate: don't crash on attic archives w/o time_end, #3109\n- don't crash on repository filesystems w/o hardlink support, #3107\n- don't crash in first part of truncate_and_unlink, #3117\n- fix server-side IndexError crash with clients < 1.0.7, #3192\n- don't show traceback if only a global option is given, show help, #3142\n- cache: use SaveFile for more safety, #3158\n- init: fix wrong encryption choices in command line parser, fix missing\n \"authenticated-blake2\", #3103\n- move --no-files-cache from common to borg create options, #3146\n- fix detection of non-local path (failed on ..filename), #3108\n- logging with fileConfig: set json attr on \"borg\" logger, #3114\n- fix crash with relative BORG_KEY_FILE, #3197\n- show excluded dir with \"x\" for tagged dirs / caches, #3189\n\nNew features:\n\n- create: --nobsdflags and --exclude-nodump options, #3160\n- extract: --nobsdflags option, #3160\n\nOther changes:\n\n- remove annoying hardlinked symlinks warning, #3175\n- vagrant: use self-made FreeBSD 10.3 box, #3022\n- travis: don't brew update, hopefully fixes #2532\n- docs:\n\n - readme: -e option is required in borg 1.1\n - add example showing --show-version --show-rc\n - use --format rather than --list-format (deprecated) in example\n - update docs about hardlinked symlinks limitation\n\n\nVersion 1.1.0 (2017-10-07)\n--------------------------\n\nCompatibility notes:\n\n- borg command line: do not put options in between positional arguments\n\n This sometimes works (e.g. it worked in borg 1.0.x), but can easily stop\n working if we make positional arguments optional (like it happened for\n borg create's \"paths\" argument in 1.1). There are also places in borg 1.0\n where we do that, so it doesn't work there in general either. #3356\n\n Good: borg create -v --stats repo::archive path\n Good: borg create repo::archive path -v --stats\n Bad: borg create repo::archive -v --stats path\n\nFixes:\n\n- fix LD_LIBRARY_PATH restoration for subprocesses, #3077\n- \"auto\" compression: make sure expensive compression is actually better,\n otherwise store lz4 compressed data we already computed.\n\nOther changes:\n\n- docs:\n\n - FAQ: we do not implement futile attempts of ETA / progress displays\n - manpage: fix typos, update homepage\n - implement simple \"issue\" role for manpage generation, #3075\n\n\nVersion 1.1.0rc4 (2017-10-01)\n-----------------------------\n\nCompatibility notes:\n\n- A borg server >= 1.1.0rc4 does not support borg clients 1.1.0b3-b5. #3033\n- The files cache is now controlled differently and has a new default mode:\n\n - the files cache now uses ctime by default for improved file change\n detection safety. You can still use mtime for more speed and less safety.\n - --ignore-inode is deprecated (use --files-cache=... without \"inode\")\n - --no-files-cache is deprecated (use --files-cache=disabled)\n\nNew features:\n\n- --files-cache - implement files cache mode control, #911\n You can now control the files cache mode using this option:\n --files-cache={ctime,mtime,size,inode,rechunk,disabled}\n (only some combinations are supported). See the docs for details.\n\nFixes:\n\n- remote progress/logging: deal with partial lines, #2637\n- remote progress: flush json mode output\n- fix subprocess environments, #3050 (and more)\n\nOther changes:\n\n- remove client_supports_log_v3 flag, #3033\n- exclude broken Cython 0.27(.0) in requirements, #3066\n- vagrant:\n\n - upgrade to FUSE for macOS 3.7.1\n - use Python 3.5.4 to build the binaries\n- docs:\n\n - security: change-passphrase only changes the passphrase, #2990\n - fixed/improved borg create --compression examples, #3034\n - add note about metadata dedup and --no[ac]time, #2518\n - twitter account @borgbackup now, better visible, #2948\n - simplified rate limiting wrapper in FAQ\n\n\nVersion 1.1.0rc3 (2017-09-10)\n-----------------------------\n\nNew features:\n\n- delete: support naming multiple archives, #2958\n\nFixes:\n\n- repo cleanup/write: invalidate cached FDs, #2982\n- fix datetime.isoformat() microseconds issues, #2994\n- recover_segment: use mmap(), lower memory needs, #2987\n\nOther changes:\n\n- with-lock: close segment file before invoking subprocess\n- keymanager: don't depend on optional readline module, #2976\n- docs:\n\n - fix macOS keychain integration command\n - show/link new screencasts in README, #2936\n - document utf-8 locale requirement for json mode, #2273\n- vagrant: clean up shell profile init, user name, #2977\n- test_detect_attic_repo: don't test mount, #2975\n- add debug logging for repository cleanup\n\n\nVersion 1.1.0rc2 (2017-08-28)\n-----------------------------\n\nCompatibility notes:\n\n- list: corrected mix-up of \"isomtime\" and \"mtime\" formats. Previously,\n \"isomtime\" was the default but produced a verbose human format,\n while \"mtime\" produced a ISO-8601-like format.\n The behaviours have been swapped (so \"mtime\" is human, \"isomtime\" is ISO-like),\n and the default is now \"mtime\".\n \"isomtime\" is now a real ISO-8601 format (\"T\" between date and time, not a space).\n\nNew features:\n\n- None.\n\nFixes:\n\n- list: fix weird mixup of mtime/isomtime\n- create --timestamp: set start time, #2957\n- ignore corrupt files cache, #2939\n- migrate locks to child PID when daemonize is used\n- fix exitcode of borg serve, #2910\n- only compare contents when chunker params match, #2899\n- umount: try fusermount, then try umount, #2863\n\nOther changes:\n\n- JSON: use a more standard ISO 8601 datetime format, #2376\n- cache: write_archive_index: truncate_and_unlink on error, #2628\n- detect non-upgraded Attic repositories, #1933\n- delete various nogil and threading related lines\n- coala / pylint related improvements\n- docs:\n\n - renew asciinema/screencasts, #669\n - create: document exclusion through nodump, #2949\n - minor formatting fixes\n - tar: tarpipe example\n - improve \"with-lock\" and \"info\" docs, #2869\n - detail how to use macOS/GNOME/KDE keyrings for repo passwords, #392\n- travis: only short-circuit docs-only changes for pull requests\n- vagrant:\n\n - netbsd: bash is already installed\n - fix netbsd version in PKG_PATH\n - add exe location to PATH when we build an exe\n\n\nVersion 1.1.0rc1 (2017-07-24)\n-----------------------------\n\nCompatibility notes:\n\n- delete: removed short option for --cache-only\n\nNew features:\n\n- support borg list repo --format {comment} {bcomment} {end}, #2081\n- key import: allow reading from stdin, #2760\n\nFixes:\n\n- with-lock: avoid creating segment files that might be overwritten later, #1867\n- prune: fix checkpoints processing with --glob-archives\n- FUSE: versions view: keep original file extension at end, #2769\n- fix --last, --first: do not accept values <= 0,\n fix reversed archive ordering with --last\n- include testsuite data (attic.tar.gz) when installing the package\n- use limited unpacker for outer key, for manifest (both security precautions),\n #2174 #2175\n- fix bashism in shell scripts, #2820, #2816\n- cleanup endianness detection, create _endian.h,\n fixes build on alpine linux, #2809\n- fix crash with --no-cache-sync (give known chunk size to chunk_incref), #2853\n\nOther changes:\n\n- FUSE: versions view: linear numbering by archive time\n- split up interval parsing from filtering for --keep-within, #2610\n- add a basic .editorconfig, #2734\n- use archive creation time as mtime for FUSE mount, #2834\n- upgrade FUSE for macOS (osxfuse) from 3.5.8 to 3.6.3, #2706\n- hashindex: speed up by replacing modulo with \"if\" to check for wraparound\n- coala checker / pylint: fixed requirements and .coafile, more ignores\n- borg upgrade: name backup directories as 'before-upgrade', #2811\n- add .mailmap\n- some minor changes suggested by lgtm.com\n- docs:\n\n - better explanation of the --ignore-inode option relevance, #2800\n - fix openSUSE command and add openSUSE section\n - simplify ssh authorized_keys file using \"restrict\", add legacy note, #2121\n - mount: show usage of archive filters\n - mount: add repository example, #2462\n - info: update and add examples, #2765\n - prune: include example\n - improved style / formatting\n - improved/fixed segments_per_dir docs\n - recreate: fix wrong \"remove unwanted files\" example\n - reference list of status chars in borg recreate --filter description\n - update source-install docs about doc build dependencies, #2795\n - cleanup installation docs\n - file system requirements, update segs per dir\n - fix checkpoints/parts reference in FAQ, #2859\n- code:\n\n - hashindex: don't pass side effect into macro\n - crypto low_level: don't mutate local bytes()\n - use dash_open function to open file or \"-\" for stdin/stdout\n - archiver: argparse cleanup / refactoring\n - shellpattern: add match_end arg\n- tests: added some additional unit tests, some fixes, #2700 #2710\n- vagrant: fix setup of cygwin, add Debian 9 \"stretch\"\n- travis: don't perform full travis build on docs-only changes, #2531\n\n\nVersion 1.1.0b6 (2017-06-18)\n----------------------------\n\nCompatibility notes:\n\n- Running \"borg init\" via a \"borg serve --append-only\" server will *not* create\n an append-only repository anymore. Use \"borg init --append-only\" to initialize\n an append-only repository.\n\n- Repositories in the \"repokey\" and \"repokey-blake2\" modes with an empty passphrase\n are now treated as unencrypted repositories for security checks (e.g.\n BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK).\n\n Previously there would be no prompts nor messages if an unknown repository\n in one of these modes with an empty passphrase was encountered. This would\n allow an attacker to swap a repository, if one assumed that the lack of\n password prompts was due to a set BORG_PASSPHRASE.\n\n Since the \"trick\" does not work if BORG_PASSPHRASE is set, this does generally\n not affect scripts.\n\n- Repositories in the \"authenticated\" mode are now treated as the unencrypted\n repositories they are.\n\n- The client-side temporary repository cache now holds unencrypted data for better speed.\n\n- borg init: removed the short form of --append-only (-a).\n\n- borg upgrade: removed the short form of --inplace (-i).\n\nNew features:\n\n- reimplemented the RepositoryCache, size-limited caching of decrypted repo\n contents, integrity checked via xxh64. #2515\n- reduced space usage of chunks.archive.d. Existing caches are migrated during\n a cache sync. #235 #2638\n- integrity checking using xxh64 for important files used by borg, #1101:\n\n - repository: index and hints files\n - cache: chunks and files caches, chunks.archive.d\n- improve cache sync speed, #1729\n- create: new --no-cache-sync option\n- add repository mandatory feature flags infrastructure, #1806\n- Verify most operations against SecurityManager. Location, manifest timestamp\n and key types are now checked for almost all non-debug commands. #2487\n- implement storage quotas, #2517\n- serve: add --restrict-to-repository, #2589\n- BORG_PASSCOMMAND: use external tool providing the key passphrase, #2573\n- borg export-tar, #2519\n- list: --json-lines instead of --json for archive contents, #2439\n- add --debug-profile option (and also \"borg debug convert-profile\"), #2473\n- implement --glob-archives/-a, #2448\n- normalize authenticated key modes for better naming consistency:\n\n - rename \"authenticated\" to \"authenticated-blake2\" (uses blake2b)\n - implement \"authenticated\" mode (uses hmac-sha256)\n\nFixes:\n\n- hashindex: read/write indices >2 GiB on 32bit systems, better error\n reporting, #2496\n- repository URLs: implement IPv6 address support and also more informative\n error message when parsing fails.\n- mount: check whether llfuse is installed before asking for passphrase, #2540\n- mount: do pre-mount checks before opening repository, #2541\n- FUSE:\n\n - fix crash if empty (None) xattr is read, #2534\n - fix read(2) caching data in metadata cache\n - fix negative uid/gid crash (fix crash when mounting archives\n of external drives made on cygwin), #2674\n - redo ItemCache, on top of object cache\n - use decrypted cache\n - remove unnecessary normpaths\n- serve: ignore --append-only when initializing a repository (borg init), #2501\n- serve: fix incorrect type of exception_short for Errors, #2513\n- fix --exclude and --exclude-from recursing into directories, #2469\n- init: don't allow creating nested repositories, #2563\n- --json: fix encryption[mode] not being the cmdline name\n- remote: propagate Error.traceback correctly\n- fix remote logging and progress, #2241\n\n - implement --debug-topic for remote servers\n - remote: restore \"Remote:\" prefix (as used in 1.0.x)\n - rpc negotiate: enable v3 log protocol only for supported clients\n - fix --progress and logging in general for remote\n- fix parse_version, add tests, #2556\n- repository: truncate segments (and also some other files) before unlinking, #2557\n- recreate: keep timestamps as in original archive, #2384\n- recreate: if single archive is not processed, exit 2\n- patterns: don't recurse with ! / --exclude for pf:, #2509\n- cache sync: fix n^2 behaviour in lookup_name\n- extract: don't write to disk with --stdout (affected non-regular-file items), #2645\n- hashindex: implement KeyError, more tests\n\nOther changes:\n\n- remote: show path in PathNotAllowed\n- consider repokey w/o passphrase == unencrypted, #2169\n- consider authenticated mode == unencrypted, #2503\n- restrict key file names, #2560\n- document follow_symlinks requirements, check libc, use stat and chown\n with follow_symlinks=False, #2507\n- support common options on the main command, #2508\n- support common options on mid-level commands (e.g. borg *key* export)\n- make --progress a common option\n- increase DEFAULT_SEGMENTS_PER_DIR to 1000\n- chunker: fix invalid use of types (function only used by tests)\n- chunker: don't do uint32_t >> 32\n- FUSE:\n\n - add instrumentation (--debug and SIGUSR1/SIGINFO)\n - reduced memory usage for repository mounts by lazily instantiating archives\n - improved archive load times\n- info: use CacheSynchronizer & HashIndex.stats_against (better performance)\n- docs:\n\n - init: document --encryption as required\n - security: OpenSSL usage\n - security: used implementations; note python libraries\n - security: security track record of OpenSSL and msgpack\n - patterns: document denial of service (regex, wildcards)\n - init: note possible denial of service with \"none\" mode\n - init: document SHA extension is supported in OpenSSL and thus SHA is\n faster on AMD Ryzen than blake2b.\n - book: use A4 format, new builder option format.\n - book: create appendices\n - data structures: explain repository compaction\n - data structures: add chunk layout diagram\n - data structures: integrity checking\n - data structures: demingle cache and repo index\n - Attic FAQ: separate section for attic stuff\n - FAQ: I get an IntegrityError or similar - what now?\n - FAQ: Can I use Borg on SMR hard drives?, #2252\n - FAQ: specify \"using inline shell scripts\"\n - add systemd warning regarding placeholders, #2543\n - xattr: document API\n - add docs/misc/borg-data-flow data flow chart\n - debugging facilities\n - README: how to help the project, #2550\n - README: add bountysource badge, #2558\n - fresh new theme + tweaking\n - logo: vectorized (PDF and SVG) versions\n - frontends: use headlines - you can link to them\n - mark --pattern, --patterns-from as experimental\n - highlight experimental features in online docs\n - remove regex based pattern examples, #2458\n - nanorst for \"borg help TOPIC\" and --help\n - split deployment\n - deployment: hosting repositories\n - deployment: automated backups to a local hard drive\n - development: vagrant, windows10 requirements\n - development: update docs remarks\n - split usage docs, #2627\n - usage: avoid bash highlight, [options] instead of \n - usage: add benchmark page\n - helpers: truncate_and_unlink doc\n - don't suggest to leak BORG_PASSPHRASE\n - internals: columnize rather long ToC [webkit fixup]\n internals: manifest & feature flags\n - internals: more HashIndex details\n - internals: fix ASCII art equations\n - internals: edited obj graph related sections a bit\n - internals: layers image + description\n - fix way too small figures in pdf\n - index: disable syntax highlight (bash)\n - improve options formatting, fix accidental block quotes\n\n- testing / checking:\n\n - add support for using coala, #1366\n - testsuite: add ArchiverCorruptionTestCase\n - do not test logger name, #2504\n - call setup_logging after destroying logging config\n - testsuite.archiver: normalise pytest.raises vs. assert_raises\n - add test for preserved intermediate folder permissions, #2477\n - key: add round-trip test\n - remove attic dependency of the tests, #2505\n - enable remote tests on cygwin\n - tests: suppress tar's future timestamp warning\n - cache sync: add more refcount tests\n - repository: add tests, including corruption tests\n\n- vagrant:\n\n - control VM cpus and pytest workers via env vars VMCPUS and XDISTN\n - update cleaning workdir\n - fix openbsd shell\n - add OpenIndiana\n\n- packaging:\n\n - binaries: don't bundle libssl\n - setup.py clean to remove compiled files\n - fail in borg package if version metadata is very broken (setuptools_scm)\n\n- repo / code structure:\n\n - create borg.algorithms and borg.crypto packages\n - algorithms: rename crc32 to checksums\n - move patterns to module, #2469\n - gitignore: complete paths for src/ excludes\n - cache: extract CacheConfig class\n - implement IntegrityCheckedFile + Detached variant, #2502 #1688\n - introduce popen_with_error_handling to handle common user errors\n\n\nVersion 1.1.0b5 (2017-04-30)\n----------------------------\n\nCompatibility notes:\n\n- BORG_HOSTNAME_IS_UNIQUE is now on by default.\n- removed --compression-from feature\n- recreate: add --recompress flag, unify --always-recompress and\n --recompress\n\nFixes:\n\n- catch exception for os.link when hardlinks are not supported, #2405\n- borg rename / recreate: expand placeholders, #2386\n- generic support for hardlinks (files, devices, FIFOs), #2324\n- extract: also create parent dir for device files, if needed, #2358\n- extract: if a hardlink master is not in the to-be-extracted subset,\n the \"x\" status was not displayed for it, #2351\n- embrace y2038 issue to support 32bit platforms: clamp timestamps to int32,\n #2347\n- verify_data: fix IntegrityError handling for defect chunks, #2442\n- allow excluding parent and including child, #2314\n\nOther changes:\n\n- refactor compression decision stuff\n- change global compression default to lz4 as well, to be consistent\n with --compression defaults.\n- placeholders: deny access to internals and other unspecified stuff\n- clearer error message for unrecognized placeholder\n- more clear exception if borg check does not help, #2427\n- vagrant: upgrade FUSE for macOS to 3.5.8, #2346\n- linux binary builds: get rid of glibc 2.13 dependency, #2430\n- docs:\n\n - placeholders: document escaping\n - serve: env vars in original commands are ignored\n - tell what kind of hardlinks we support\n - more docs about compression\n - LICENSE: use canonical formulation\n (\"copyright holders and contributors\" instead of \"author\")\n - document borg init behaviour via append-only borg serve, #2440\n - be clear about what buzhash is used for, #2390\n - add hint about chunker params, #2421\n - clarify borg upgrade docs, #2436\n - FAQ to explain warning when running borg check --repair, #2341\n - repository file system requirements, #2080\n - pre-install considerations\n - misc. formatting / crossref fixes\n- tests:\n\n - enhance travis setuptools_scm situation\n - add extra test for the hashindex\n - fix invalid param issue in benchmarks\n\nThese belong to 1.1.0b4 release, but did not make it into changelog by then:\n\n- vagrant: increase memory for parallel testing\n- lz4 compress: lower max. buffer size, exception handling\n- add docstring to do_benchmark_crud\n- patterns help: mention path full-match in intro\n\n\nVersion 1.1.0b4 (2017-03-27)\n----------------------------\n\nCompatibility notes:\n\n- init: the --encryption argument is mandatory now (there are several choices)\n- moved \"borg migrate-to-repokey\" to \"borg key migrate-to-repokey\".\n- \"borg change-passphrase\" is deprecated, use \"borg key change-passphrase\"\n instead.\n- the --exclude-if-present option now supports tagging a folder with any\n filesystem object type (file, folder, etc), instead of expecting only files\n as tags, #1999\n- the --keep-tag-files option has been deprecated in favor of the new\n --keep-exclude-tags, to account for the change mentioned above.\n- use lz4 compression by default, #2179\n\nNew features:\n\n- JSON API to make developing frontends and automation easier\n (see :ref:`json_output`)\n\n - add JSON output to commands: `borg create/list/info --json ...`.\n - add --log-json option for structured logging output.\n - add JSON progress information, JSON support for confirmations (yes()).\n- add two new options --pattern and --patterns-from as discussed in #1406\n- new path full match pattern style (pf:) for very fast matching, #2334\n- add 'debug dump-manifest' and 'debug dump-archive' commands\n- add 'borg benchmark crud' command, #1788\n- new 'borg delete --force --force' to delete severely corrupted archives, #1975\n- info: show utilization of maximum archive size, #1452\n- list: add dsize and dcsize keys, #2164\n- paperkey.html: Add interactive html template for printing key backups.\n- key export: add qr html export mode\n- securely erase config file (which might have old encryption key), #2257\n- archived file items: add size to metadata, 'borg extract' and 'borg check' do\n check the file size for consistency, FUSE uses precomputed size from Item.\n\nFixes:\n\n- fix remote speed regression introduced in 1.1.0b3, #2185\n- fix regression handling timestamps beyond 2262 (revert bigint removal),\n introduced in 1.1.0b3, #2321\n- clamp (nano)second values to unproblematic range, #2304\n- hashindex: rebuild hashtable if we have too little empty buckets\n (performance fix), #2246\n- Location regex: fix bad parsing of wrong syntax\n- ignore posix_fadvise errors in repository.py, #2095\n- borg rpc: use limited msgpack.Unpacker (security precaution), #2139\n- Manifest: Make sure manifest timestamp is strictly monotonically increasing.\n- create: handle BackupOSError on a per-path level in one spot\n- create: clarify -x option / meaning of \"same filesystem\"\n- create: don't create hard link refs to failed files\n- archive check: detect and fix missing all-zero replacement chunks, #2180\n- files cache: update inode number when --ignore-inode is used, #2226\n- fix decompression exceptions crashing ``check --verify-data`` and others\n instead of reporting integrity error, #2224 #2221\n- extract: warning for unextracted big extended attributes, #2258, #2161\n- mount: umount on SIGINT/^C when in foreground\n- mount: handle invalid hard link refs\n- mount: fix huge RAM consumption when mounting a repository (saves number of\n archives * 8 MiB), #2308\n- hashindex: detect mingw byte order #2073\n- hashindex: fix wrong skip_hint on hashindex_set when encountering tombstones,\n the regression was introduced in #1748\n- fix ChunkIndex.__contains__ assertion for big-endian archs\n- fix borg key/debug/benchmark crashing without subcommand, #2240\n- Location: accept //servername/share/path\n- correct/refactor calculation of unique/non-unique chunks\n- extract: fix missing call to ProgressIndicator.finish\n- prune: fix error msg, it is --keep-within, not --within\n- fix \"auto\" compression mode bug (not compressing), #2331\n- fix symlink item fs size computation, #2344\n\nOther changes:\n\n- remote repository: improved async exception processing, #2255 #2225\n- with --compression auto,C, only use C if lz4 achieves at least 3% compression\n- PatternMatcher: only normalize path once, #2338\n- hashindex: separate endian-dependent defs from endian detection\n- migrate-to-repokey: ask using canonical_path() as we do everywhere else.\n- SyncFile: fix use of fd object after close\n- make LoggedIO.close_segment reentrant\n- creating a new segment: use \"xb\" mode, #2099\n- redo key_creator, key_factory, centralise key knowledge, #2272\n- add return code functions, #2199\n- list: only load cache if needed\n- list: files->items, clarifications\n- list: add \"name\" key for consistency with info cmd\n- ArchiveFormatter: add \"start\" key for compatibility with \"info\"\n- RemoteRepository: account rx/tx bytes\n- setup.py build_usage/build_man/build_api fixes\n- Manifest.in: simplify, exclude .so, .dll and .orig, #2066\n- FUSE: get rid of chunk accounting, st_blocks = ceil(size / blocksize).\n- tests:\n\n - help python development by testing 3.6-dev\n - test for borg delete --force\n- vagrant:\n\n - freebsd: some fixes, #2067\n - darwin64: use osxfuse 3.5.4 for tests / to build binaries\n - darwin64: improve VM settings\n - use python 3.5.3 to build binaries, #2078\n - upgrade pyinstaller from 3.1.1+ to 3.2.1\n - pyinstaller: use fixed AND freshly compiled bootloader, #2002\n - pyinstaller: automatically builds bootloader if missing\n- docs:\n\n - create really nice man pages\n - faq: mention --remote-ratelimit in bandwidth limit question\n - fix caskroom link, #2299\n - docs/security: reiterate that RPC in Borg does no networking\n - docs/security: counter tracking, #2266\n - docs/development: update merge remarks\n - address SSH batch mode in docs, #2202 #2270\n - add warning about running build_usage on Python >3.4, #2123\n - one link per distro in the installation page\n - improve --exclude-if-present and --keep-exclude-tags, #2268\n - improve automated backup script in doc, #2214\n - improve remote-path description\n - update docs for create -C default change (lz4)\n - document relative path usage, #1868\n - document snapshot usage, #2178\n - corrected some stuff in internals+security\n - internals: move toctree to after the introduction text\n - clarify metadata kind, manifest ops\n - key enc: correct / clarify some stuff, link to internals/security\n - datas: enc: 1.1.x mas different MACs\n - datas: enc: correct factual error -- no nonce involved there.\n - make internals.rst an index page and edit it a bit\n - add \"Cryptography in Borg\" and \"Remote RPC protocol security\" sections\n - document BORG_HOSTNAME_IS_UNIQUE, #2087\n - FAQ by categories as proposed by @anarcat in #1802\n - FAQ: update Which file types, attributes, etc. are *not* preserved?\n - development: new branching model for git repository\n - development: define \"ours\" merge strategy for auto-generated files\n - create: move --exclude note to main doc\n - create: move item flags to main doc\n - fix examples using borg init without -e/--encryption\n - list: don't print key listings in fat (html + man)\n - remove Python API docs (were very incomplete, build problems on RTFD)\n - added FAQ section about backing up root partition\n\n\nVersion 1.1.0b3 (2017-01-15)\n----------------------------\n\nCompatibility notes:\n\n- borg init: removed the default of \"--encryption/-e\", #1979\n This was done so users do a informed decision about -e mode.\n\nBug fixes:\n\n- borg recreate: don't rechunkify unless explicitly told so\n- borg info: fixed bug when called without arguments, #1914\n- borg init: fix free space check crashing if disk is full, #1821\n- borg debug delete/get obj: fix wrong reference to exception\n- fix processing of remote ~/ and ~user/ paths (regressed since 1.1.0b1), #1759\n- posix platform module: only build / import on non-win32 platforms, #2041\n\nNew features:\n\n- new CRC32 implementations that are much faster than the zlib one used previously, #1970\n- add blake2b key modes (use blake2b as MAC). This links against system libb2,\n if possible, otherwise uses bundled code\n- automatically remove stale locks - set BORG_HOSTNAME_IS_UNIQUE env var\n to enable stale lock killing. If set, stale locks in both cache and\n repository are deleted. #562 #1253\n- borg info : print general repo information, #1680\n- borg check --first / --last / --sort / --prefix, #1663\n- borg mount --first / --last / --sort / --prefix, #1542\n- implement \"health\" item formatter key, #1749\n- BORG_SECURITY_DIR to remember security related infos outside the cache.\n Key type, location and manifest timestamp checks now survive cache\n deletion. This also means that you can now delete your cache and avoid\n previous warnings, since Borg can still tell it's safe.\n- implement BORG_NEW_PASSPHRASE, #1768\n\nOther changes:\n\n- borg recreate:\n\n - remove special-cased --dry-run\n - update --help\n - remove bloat: interruption blah, autocommit blah, resuming blah\n - re-use existing checkpoint functionality\n - archiver tests: add check_cache tool - lints refcounts\n\n- fixed cache sync performance regression from 1.1.0b1 onwards, #1940\n- syncing the cache without chunks.archive.d\n now avoids any merges and is thus faster, #1940\n- borg check --verify-data: faster due to linear on-disk-order scan\n- borg debug-xxx commands removed, we use \"debug xxx\" subcommands now, #1627\n- improve metadata handling speed\n- shortcut hashindex_set by having hashindex_lookup hint about address\n- improve / add progress displays, #1721\n- check for index vs. segment files object count mismatch\n- make RPC protocol more extensible: use named parameters.\n- RemoteRepository: misc. code cleanups / refactors\n- clarify cache/repository README file\n\n- docs:\n\n - quickstart: add a comment about other (remote) filesystems\n - quickstart: only give one possible ssh url syntax, all others are\n documented in usage chapter.\n - mention file://\n - document repo URLs / archive location\n - clarify borg diff help, #980\n - deployment: synthesize alternative --restrict-to-path example\n - improve cache / index docs, esp. files cache docs, #1825\n - document using \"git merge 1.0-maint -s recursive -X rename-threshold=20%\"\n for avoiding troubles when merging the 1.0-maint branch into master.\n\n- tests:\n\n - FUSE tests: catch ENOTSUP on freebsd\n - FUSE tests: test troublesome xattrs last\n - fix byte range error in test, #1740\n - use monkeypatch to set env vars, but only on pytest based tests.\n - point XDG_*_HOME to temp dirs for tests, #1714\n - remove all BORG_* env vars from the outer environment\n\n\nVersion 1.1.0b2 (2016-10-01)\n----------------------------\n\nBug fixes:\n\n- fix incorrect preservation of delete tags, leading to \"object count mismatch\"\n on borg check, #1598. This only occurred with 1.1.0b1 (not with 1.0.x) and is\n normally fixed by running another borg create/delete/prune.\n- fix broken --progress for double-cell paths (e.g. CJK), #1624\n- borg recreate: also catch SIGHUP\n- FUSE:\n\n - fix hardlinks in versions view, #1599\n - add parameter check to ItemCache.get to make potential failures more clear\n\nNew features:\n\n- Archiver, RemoteRepository: add --remote-ratelimit (send data)\n- borg help compression, #1582\n- borg check: delete chunks with integrity errors, #1575, so they can be\n \"repaired\" immediately and maybe healed later.\n- archives filters concept (refactoring/unifying older code)\n\n - covers --first/--last/--prefix/--sort-by options\n - currently used for borg list/info/delete\n\nOther changes:\n\n- borg check --verify-data slightly tuned (use get_many())\n- change {utcnow} and {now} to ISO-8601 format (\"T\" date/time separator)\n- repo check: log transaction IDs, improve object count mismatch diagnostic\n- Vagrantfile: use TW's fresh-bootloader pyinstaller branch\n- fix module names in api.rst\n- hashindex: bump api_version\n\n\nVersion 1.1.0b1 (2016-08-28)\n----------------------------\n\nNew features:\n\n- new commands:\n\n - borg recreate: re-create existing archives, #787 #686 #630 #70, also see\n #757, #770.\n\n - selectively remove files/dirs from old archives\n - re-compress data\n - re-chunkify data, e.g. to have upgraded Attic / Borg 0.xx archives\n deduplicate with Borg 1.x archives or to experiment with chunker-params.\n - borg diff: show differences between archives\n - borg with-lock: execute a command with the repository locked, #990\n- borg create:\n\n - Flexible compression with pattern matching on path/filename,\n and LZ4 heuristic for deciding compressibility, #810, #1007\n - visit files in inode order (better speed, esp. for large directories and rotating disks)\n - in-file checkpoints, #1217\n - increased default checkpoint interval to 30 minutes (was 5 minutes), #896\n - added uuid archive format tag, #1151\n - save mountpoint directories with --one-file-system, makes system restore easier, #1033\n - Linux: added support for some BSD flags, #1050\n - add 'x' status for excluded paths, #814\n\n - also means files excluded via UF_NODUMP, #1080\n- borg check:\n\n - will not produce the \"Checking segments\" output unless new --progress option is passed, #824.\n - --verify-data to verify data cryptographically on the client, #975\n- borg list, #751, #1179\n\n - removed {formatkeys}, see \"borg list --help\"\n - --list-format is deprecated, use --format instead\n - --format now also applies to listing archives, not only archive contents, #1179\n - now supports the usual [PATH [PATHS…]] syntax and excludes\n - new keys: csize, num_chunks, unique_chunks, NUL\n - supports guaranteed_available hashlib hashes\n (to avoid varying functionality depending on environment),\n which includes the SHA1 and SHA2 family as well as MD5\n- borg prune:\n\n - to visualize the \"thinning out\" better, we now list all archives in\n reverse time order. rephrase and reorder help text.\n - implement --keep-last N via --keep-secondly N, also --keep-minutely.\n assuming that there is not more than 1 backup archive made in 1s,\n --keep-last N and --keep-secondly N are equivalent, #537\n - cleanup checkpoints except the latest, #1008\n- borg extract:\n\n - added --progress, #1449\n - Linux: limited support for BSD flags, #1050\n- borg info:\n\n - output is now more similar to borg create --stats, #977\n- borg mount:\n\n - provide \"borgfs\" wrapper for borg mount, enables usage via fstab, #743\n - \"versions\" mount option - when used with a repository mount, this gives\n a merged, versioned view of the files in all archives, #729\n- repository:\n\n - added progress information to commit/compaction phase (often takes some time when deleting/pruning), #1519\n - automatic recovery for some forms of repository inconsistency, #858\n - check free space before going forward with a commit, #1336\n - improved write performance (esp. for rotating media), #985\n\n - new IO code for Linux\n - raised default segment size to approx 512 MiB\n - improved compaction performance, #1041\n - reduced client CPU load and improved performance for remote repositories, #940\n\n- options that imply output (--show-rc, --show-version, --list, --stats,\n --progress) don't need -v/--info to have that output displayed, #865\n- add archive comments (via borg (re)create --comment), #842\n- borg list/prune/delete: also output archive id, #731\n- --show-version: shows/logs the borg version, #725\n- added --debug-topic for granular debug logging, #1447\n- use atomic file writing/updating for configuration and key files, #1377\n- BORG_KEY_FILE environment variable, #1001\n- self-testing module, #970\n\n\nBug fixes:\n\n- list: fixed default output being produced if --format is given with empty parameter, #1489\n- create: fixed overflowing progress line with CJK and similar characters, #1051\n- prune: fixed crash if --prefix resulted in no matches, #1029\n- init: clean up partial repo if passphrase input is aborted, #850\n- info: quote cmdline arguments that have spaces in them\n- fix hardlinks failing in some cases for extracting subtrees, #761\n\nOther changes:\n\n- replace stdlib hmac with OpenSSL, zero-copy decrypt (10-15% increase in\n performance of hash-lists and extract).\n- improved chunker performance, #1021\n- open repository segment files in exclusive mode (fail-safe), #1134\n- improved error logging, #1440\n- Source:\n\n - pass meta-data around, #765\n - move some constants to new constants module\n - better readability and fewer errors with namedtuples, #823\n - moved source tree into src/ subdirectory, #1016\n - made borg.platform a package, #1113\n - removed dead crypto code, #1032\n - improved and ported parts of the test suite to py.test, #912\n - created data classes instead of passing dictionaries around, #981, #1158, #1161\n - cleaned up imports, #1112\n- Docs:\n\n - better help texts and sphinx reproduction of usage help:\n\n - Group options\n - Nicer list of options in Sphinx\n - Deduplicate 'Common options' (including --help)\n - chunker: added some insights by \"Voltara\", #903\n - clarify what \"deduplicated size\" means\n - fix / update / add package list entries\n - added a SaltStack usage example, #956\n - expanded FAQ\n - new contributors in AUTHORS!\n- Tests:\n\n - vagrant: add ubuntu/xenial 64bit - this box has still some issues\n - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945\n\n\nVersion 1.0.13 (2019-02-15)\n---------------------------\n\nPlease note: this is very likely the last 1.0.x release, please upgrade to 1.1.x.\n\nBug fixes:\n\n- security fix: configure FUSE with \"default_permissions\", #3903.\n \"default_permissions\" is now enforced by borg by default to let the\n kernel check uid/gid/mode based permissions.\n \"ignore_permissions\" can be given not to enforce \"default_permissions\".\n- xattrs: fix borg exception handling on ENOSPC error, #3808.\n\nNew features:\n\n- Read a passphrase from a file descriptor specified in the\n BORG_PASSPHRASE_FD environment variable.\n\nOther changes:\n\n- acl platform code: fix acl set return type\n- xattr:\n\n - add linux {list,get,set}xattr ctypes prototypes\n - fix darwin flistxattr ctypes prototype\n- testing / travis-ci:\n\n - fix the homebrew 1.9 issues on travis-ci, #4254\n - travis OS X: use xcode 8.3 (not broken)\n - tox.ini: lock requirements\n - unbreak 1.0-maint on travis, fixes #4123\n- vagrant:\n\n - misc. fixes\n - FUSE for macOS: upgrade 3.7.1 to 3.8.3\n - Python: upgrade 3.5.5 to 3.5.6\n- docs:\n\n - Update installation instructions for macOS\n - update release workflow using twine (docs, scripts), #4213\n\nVersion 1.0.12 (2018-04-08)\n---------------------------\n\nBug fixes:\n\n- repository: cleanup/write: invalidate cached FDs, tests\n- serve: fix exitcode, #2910\n- extract: set bsdflags last (include immutable flag), #3263\n- create --timestamp: set start time, #2957\n- create: show excluded dir with \"x\" for tagged dirs / caches, #3189\n- migrate locks to child PID when daemonize is used\n- Buffer: fix wrong thread-local storage use, #2951\n- fix detection of non-local path, #3108\n- fix LDLP restoration for subprocesses, #3077\n- fix subprocess environments (xattr module's fakeroot version check,\n borg umount, BORG_PASSCOMMAND), #3050\n- remote: deal with partial lines, #2637\n- get rid of datetime.isoformat, use safe parse_timestamp to parse\n timestamps, #2994\n- build: do .h file content checks in binary mode, fixes build issue for\n non-ascii header files on pure-ascii locale platforms, #3544 #3639\n- remove platform.uname() call which caused library mismatch issues, #3732\n- add exception handler around deprecated platform.linux_distribution() call\n\nOther changes:\n\n- require msgpack-python >= 0.4.6 and < 0.5.0, see #3753\n- add parens for C preprocessor macro argument usages (did not cause\n malfunction)\n- ignore corrupt files cache, #2939\n- replace \"modulo\" with \"if\" to check for wraparound in hashmap\n- keymanager: don't depend on optional readline module, #2980\n- exclude broken pytest 3.3.0 release\n- exclude broken Cython 0.27(.0) release, #3066\n- flake8: add some ignores\n- docs:\n\n - create: document exclusion through nodump\n - document good and problematic option placements, fix examples, #3356\n - update docs about hardlinked symlinks limitation\n - faq: we do not implement futile attempts of ETA / progress displays\n - simplified rate limiting wrapper in FAQ\n - twitter account @borgbackup, #2948\n - add note about metadata dedup and --no[ac]time, #2518\n - change-passphrase only changes the passphrase, #2990\n - clarify encrypted key format for borg key export, #3296\n - document sshfs rename workaround, #3315\n - update release checklist about security fixes\n - docs about how to verify a signed release, #3634\n - chunk seed is generated per /repository/\n- vagrant:\n\n - use FUSE for macOS 3.7.1 to build the macOS binary\n - use python 3.5.5 to build the binaries\n - add exe location to PATH when we build an exe\n - use https pypi url for wheezy\n - netbsd: bash is already installed\n - netbsd: fix netbsd version in PKG_PATH\n - use self-made FreeBSD 10.3 box, #3022\n - backport fs_init (including related updates) from 1.1\n - the boxcutter wheezy boxes are 404, use local ones\n- travis:\n\n - don't perform full Travis build on docs-only changes, #2531\n - only short-circuit docs-only changes for pull requests\n\n\nVersion 1.0.11 (2017-07-21)\n---------------------------\n\nBug fixes:\n\n- use limited unpacker for outer key (security precaution), #2174\n- fix paperkey import bug\n\nOther changes:\n\n- change --checkpoint-interval default from 600s to 1800s, #2841.\n this improves efficiency for big repositories a lot.\n- docs: fix OpenSUSE command and add OpenSUSE section\n- tests: add tests for split_lstring and paperkey\n- vagrant:\n\n - fix openbsd shell\n - backport cpu/ram setup from master\n - add stretch64 VM\n\nVersion 1.0.11rc1 (2017-06-27)\n------------------------------\n\nBug fixes:\n\n- performance: rebuild hashtable if we have too few empty buckets, #2246.\n this fixes some sporadic, but severe performance breakdowns.\n- Archive: allocate zeros when needed, #2308\n fixes huge memory usage of mount (8 MiB × number of archives)\n- IPv6 address support\n also: Location: more informative exception when parsing fails\n- borg single-file binary: use pyinstaller v3.2.1, #2396\n this fixes that the prelink cronjob on some distros kills the\n borg binary by stripping away parts of it.\n- extract:\n\n - warning for unextracted big extended attributes, #2258\n - also create parent dir for device files, if needed.\n - don't write to disk with --stdout, #2645\n- archive check: detect and fix missing all-zero replacement chunks, #2180\n- fix (de)compression exceptions, #2224 #2221\n- files cache: update inode number, #2226\n- borg rpc: use limited msgpack.Unpacker (security precaution), #2139\n- Manifest: use limited msgpack.Unpacker (security precaution), #2175\n- Location: accept //servername/share/path\n- fix ChunkIndex.__contains__ assertion for big-endian archs (harmless)\n- create: handle BackupOSError on a per-path level in one spot\n- fix error msg, there is no --keep-last in borg 1.0.x, #2282\n- clamp (nano)second values to unproblematic range, #2304\n- fuse / borg mount:\n\n - fix st_blocks to be an integer (not float) value\n - fix negative uid/gid crash (they could come into archives e.g. when\n backing up external drives under cygwin), #2674\n - fix crash if empty (None) xattr is read\n - do pre-mount checks before opening repository\n - check llfuse is installed before asking for passphrase\n- borg rename: expand placeholders, #2386\n- borg serve: fix forced command lines containing BORG_* env vars\n- fix error msg, it is --keep-within, not --within\n- fix borg key/debug/benchmark crashing without subcommand, #2240\n- chunker: fix invalid use of types, don't do uint32_t >> 32\n- document follow_symlinks requirements, check libc, #2507\n\nNew features:\n\n- added BORG_PASSCOMMAND environment variable, #2573\n- add minimal version of in repository mandatory feature flags, #2134\n\n This should allow us to make sure older borg versions can be cleanly\n prevented from doing operations that are no longer safe because of\n repository format evolution. This allows more fine grained control than\n just incrementing the manifest version. So for example a change that\n still allows new archives to be created but would corrupt the repository\n when an old version tries to delete an archive or check the repository\n would add the new feature to the check and delete set but leave it out\n of the write set.\n- borg delete --force --force to delete severely corrupted archives, #1975\n\nOther changes:\n\n- embrace y2038 issue to support 32bit platforms\n- be more clear that this is a \"beyond repair\" case, #2427\n- key file names: limit to 100 characters and remove colons from host name\n- upgrade FUSE for macOS to 3.5.8, #2346\n- split up parsing and filtering for --keep-within, better error message, #2610\n- docs:\n\n - fix caskroom link, #2299\n - address SSH batch mode, #2202 #2270\n - improve remote-path description\n - document snapshot usage, #2178\n - document relative path usage, #1868\n - one link per distro in the installation page\n - development: new branching model in git repository\n - kill api page\n - added FAQ section about backing up root partition\n - add bountysource badge, #2558\n - create empty docs.txt requirements, #2694\n - README: how to help the project\n - note -v/--verbose requirement on affected options, #2542\n - document borg init behaviour via append-only borg serve, #2440\n - be clear about what buzhash is used for (chunking) and want it is not\n used for (deduplication)- also say already in the readme that we use a\n cryptohash for dedupe, so people don't worry, #2390\n - add hint about chunker params to borg upgrade docs, #2421\n - clarify borg upgrade docs, #2436\n - quickstart: delete problematic BORG_PASSPHRASE use, #2623\n - faq: specify \"using inline shell scripts\"\n - document pattern denial of service, #2624\n- tests:\n\n - remove attic dependency of the tests, #2505\n - travis:\n\n - enhance travis setuptools_scm situation\n - install fakeroot for Linux\n - add test for borg delete --force\n - enable remote tests on cygwin (the cygwin issue that caused these tests\n to break was fixed in cygwin at least since cygwin 2.8, maybe even since\n 2.7.0).\n - remove skipping the noatime tests on GNU/Hurd, #2710\n - fix borg import issue, add comment, #2718\n - include attic.tar.gz when installing the package\n also: add include_package_data=True\n\nVersion 1.0.10 (2017-02-13)\n---------------------------\n\nBug fixes:\n\n- Manifest timestamps are now monotonically increasing,\n this fixes issues when the system clock jumps backwards\n or is set inconsistently across computers accessing the same repository, #2115\n- Fixed testing regression in 1.0.10rc1 that lead to a hard dependency on\n py.test >= 3.0, #2112\n\nNew features:\n\n- \"key export\" can now generate a printable HTML page with both a QR code and\n a human-readable \"paperkey\" representation (and custom text) through the\n ``--qr-html`` option.\n\n The same functionality is also available through `paperkey.html `_,\n which is the same HTML page generated by ``--qr-html``. It works with existing\n \"key export\" files and key files.\n\nOther changes:\n\n- docs:\n\n - language clarification - \"borg create --one-file-system\" option does not respect\n mount points, but considers different file systems instead, #2141\n- setup.py: build_api: sort file list for determinism\n\n\nVersion 1.0.10rc1 (2017-01-29)\n------------------------------\n\nBug fixes:\n\n- borg serve: fix transmission data loss of pipe writes, #1268\n This affects only the cygwin platform (not Linux, BSD, OS X).\n- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992\n- When running out of buffer memory when reading xattrs, only skip the\n current file, #1993\n- Fixed \"borg upgrade --tam\" crashing with unencrypted repositories. Since\n :ref:`the issue ` is not relevant for unencrypted repositories,\n it now does nothing and prints an error, #1981.\n- Fixed change-passphrase crashing with unencrypted repositories, #1978\n- Fixed \"borg check repo::archive\" indicating success if \"archive\" does not exist, #1997\n- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled\n- fix bad parsing of wrong repo location syntax\n- create: don't create hard link refs to failed files,\n mount: handle invalid hard link refs, #2092\n- detect mingw byte order, #2073\n- creating a new segment: use \"xb\" mode, #2099\n- mount: umount on SIGINT/^C when in foreground, #2082\n\nOther changes:\n\n- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002\n- xattr: ignore empty names returned by llistxattr(2) et al\n- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT,\n SIGBUS and SIGILL signals to dump the Python traceback.\n- Also print a traceback on SIGUSR2.\n- borg change-passphrase: print key location (simplify making a backup of it)\n- officially support Python 3.6 (setup.py: add Python 3.6 qualifier)\n- tests:\n\n - vagrant / travis / tox: add Python 3.6 based testing\n - vagrant: fix openbsd repo, #2042\n - vagrant: fix the freebsd64 machine, #2037 #2067\n - vagrant: use python 3.5.3 to build binaries, #2078\n - vagrant: use osxfuse 3.5.4 for tests / to build binaries\n vagrant: improve darwin64 VM settings\n - travis: fix osxfuse install (fixes OS X testing on Travis CI)\n - travis: require succeeding OS X tests, #2028\n - travis: use latest pythons for OS X based testing\n - use pytest-xdist to parallelize testing\n - fix xattr test race condition, #2047\n - setup.cfg: fix pytest deprecation warning, #2050\n- docs:\n\n - language clarification - VM backup FAQ\n - borg create: document how to back up stdin, #2013\n - borg upgrade: fix incorrect title levels\n - add CVE numbers for issues fixed in 1.0.9, #2106\n- fix typos (taken from Debian package patch)\n- remote: include data hexdump in \"unexpected RPC data\" error message\n- remote: log SSH command line at debug level\n- API_VERSION: use numberspaces, #2023\n- remove .github from pypi package, #2051\n- add pip and setuptools to requirements file, #2030\n- SyncFile: fix use of fd object after close (cosmetic)\n- Manifest.in: simplify, exclude \\*.{so,dll,orig}, #2066\n- ignore posix_fadvise errors in repository.py, #2095\n (works around issues with docker on ARM)\n- make LoggedIO.close_segment reentrant, avoid reentrance\n\n\nVersion 1.0.9 (2016-12-20)\n--------------------------\n\nSecurity fixes:\n\n- A flaw in the cryptographic authentication scheme in Borg allowed an attacker\n to spoof the manifest. See :ref:`tam_vuln` above for the steps you should\n take.\n\n CVE-2016-10099 was assigned to this vulnerability.\n- borg check: When rebuilding the manifest (which should only be needed very rarely)\n duplicate archive names would be handled on a \"first come first serve\" basis,\n potentially opening an attack vector to replace archives.\n\n Example: were there 2 archives named \"foo\" in a repo (which can not happen\n under normal circumstances, because borg checks if the name is already used)\n and a \"borg check\" recreated a (previously lost) manifest, the first of the\n archives it encountered would be in the manifest. The second archive is also\n still in the repo, but not referenced in the manifest, in this case. If the\n second archive is the \"correct\" one (and was previously referenced from the\n manifest), it looks like it got replaced by the first one. In the manifest,\n it actually got replaced. Both remain in the repo but the \"correct\" one is no\n longer accessible via normal means - the manifest.\n\n CVE-2016-10100 was assigned to this vulnerability.\n\nBug fixes:\n\n- borg check:\n\n - rebuild manifest if it's corrupted\n - skip corrupted chunks during manifest rebuild\n- fix TypeError in integrity error handler, #1903, #1894\n- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930\n- fix wrong duration/timestamps if system clock jumped during a create\n- fix progress display not updating if system clock jumps backwards\n- fix checkpoint interval being incorrect if system clock jumps\n\nOther changes:\n\n- docs:\n\n - add python3-devel as a dependency for cygwin-based installation\n - clarify extract is relative to current directory\n - FAQ: fix link to changelog\n - markup fixes\n- tests:\n\n - test_get\\_(cache|keys)_dir: clean env state, #1897\n - get back pytest's pretty assertion failures, #1938\n- setup.py build_usage:\n\n - fixed build_usage not processing all commands\n - fixed build_usage not generating includes for debug commands\n\n\nVersion 1.0.9rc1 (2016-11-27)\n-----------------------------\n\nBug fixes:\n\n- files cache: fix determination of newest mtime in backup set (which is\n used in cache cleanup and led to wrong \"A\" [added] status for unchanged\n files in next backup), #1860.\n\n- borg check:\n\n - fix incorrectly reporting attic 0.13 and earlier archives as corrupt\n - handle repo w/o objects gracefully and also bail out early if repo is\n *completely* empty, #1815.\n- fix tox/pybuild in 1.0-maint\n- at xattr module import time, loggers are not initialized yet\n\nNew features:\n\n- borg umount \n exposed already existing umount code via the CLI api, so users can use it,\n which is more consistent than using borg to mount and fusermount -u (or\n umount) to un-mount, #1855.\n- implement borg create --noatime --noctime, fixes #1853\n\nOther changes:\n\n- docs:\n\n - display README correctly on PyPI\n - improve cache / index docs, esp. files cache docs, fixes #1825\n - different pattern matching for --exclude, #1779\n - datetime formatting examples for {now} placeholder, #1822\n - clarify passphrase mode attic repo upgrade, #1854\n - clarify --umask usage, #1859\n - clarify how to choose PR target branch\n - clarify prune behavior for different archive contents, #1824\n - fix PDF issues, add logo, fix authors, headings, TOC\n - move security verification to support section\n - fix links in standalone README (:ref: tags)\n - add link to security contact in README\n - add FAQ about security\n - move fork differences to FAQ\n - add more details about resource usage\n- tests: skip remote tests on cygwin, #1268\n- travis:\n\n - allow OS X failures until the brew cask osxfuse issue is fixed\n - caskroom osxfuse-beta gone, it's osxfuse now (3.5.3)\n- vagrant:\n\n - upgrade OSXfuse / FUSE for macOS to 3.5.3\n - remove llfuse from tox.ini at a central place\n - do not try to install llfuse on centos6\n - fix FUSE test for darwin, #1546\n - add windows virtual machine with cygwin\n - Vagrantfile cleanup / code deduplication\n\n\nVersion 1.0.8 (2016-10-29)\n--------------------------\n\nBug fixes:\n\n- RemoteRepository: Fix busy wait in call_many, #940\n\nNew features:\n\n- implement borgmajor/borgminor/borgpatch placeholders, #1694\n {borgversion} was already there (full version string). With the new\n placeholders you can now also get e.g. 1 or 1.0 or 1.0.8.\n\nOther changes:\n\n- avoid previous_location mismatch, #1741\n\n due to the changed canonicalization for relative paths in PR #1711 / #1655\n (implement /./ relpath hack), there would be a changed repo location warning\n and the user would be asked if this is ok. this would break automation and\n require manual intervention, which is unwanted.\n\n thus, we automatically fix the previous_location config entry, if it only\n changed in the expected way, but still means the same location.\n\n- docs:\n\n - deployment.rst: do not use bare variables in ansible snippet\n - add clarification about append-only mode, #1689\n - setup.py: add comment about requiring llfuse, #1726\n - update usage.rst / api.rst\n - repo url / archive location docs + typo fix\n - quickstart: add a comment about other (remote) filesystems\n\n- vagrant / tests:\n\n - no chown when rsyncing (fixes boxes w/o vagrant group)\n - fix FUSE permission issues on linux/freebsd, #1544\n - skip FUSE test for borg binary + fakeroot\n - ignore security.selinux xattrs, fixes tests on centos, #1735\n\n\nVersion 1.0.8rc1 (2016-10-17)\n-----------------------------\n\nBug fixes:\n\n- fix signal handling (SIGINT, SIGTERM, SIGHUP), #1620 #1593\n Fixes e.g. leftover lock files for quickly repeated signals (e.g. Ctrl-C\n Ctrl-C) or lost connections or systemd sending SIGHUP.\n- progress display: adapt formatting to narrow screens, do not crash, #1628\n- borg create --read-special - fix crash on broken symlink, #1584.\n also correctly processes broken symlinks. before this regressed to a crash\n (5b45385) a broken symlink would've been skipped.\n- process_symlink: fix missing backup_io()\n Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting\n dirents and dispatching to process_symlink.\n- yes(): abort on wrong answers, saying so, #1622\n- fixed exception borg serve raised when connection was closed before repository\n was opened. Add an error message for this.\n- fix read-from-closed-FD issue, #1551\n (this seems not to get triggered in 1.0.x, but was discovered in master)\n- hashindex: fix iterators (always raise StopIteration when exhausted)\n (this seems not to get triggered in 1.0.x, but was discovered in master)\n- enable relative paths in ssh:// repo URLs, via /./relpath hack, #1655\n- allow repo paths with colons, #1705\n- update changed repo location immediately after acceptance, #1524\n- fix debug get-obj / delete-obj crash if object not found and remote repo,\n #1684\n- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib\n on Mac OS X (avoids mismatch lib <-> driver), #1619\n\nNew features:\n\n- add \"borg key export\" / \"borg key import\" commands, #1555, so users are able\n to back up / restore their encryption keys more easily.\n\n Supported formats are the keyfile format used by borg internally and a\n special \"paper\" format with by line checksums for printed backups. For the\n paper format, the import is an interactive process which checks each line as\n soon as it is input.\n- add \"borg debug-refcount-obj\" to determine a repo objects' referrer counts,\n #1352\n\nOther changes:\n\n- add \"borg debug ...\" subcommands\n (borg debug-* still works, but will be removed in borg 1.1)\n- setup.py: Add subcommand support to build_usage.\n- remote: change exception message for unexpected RPC data format to indicate\n dataflow direction.\n- improved messages / error reporting:\n\n - IntegrityError: add placeholder for message, so that the message we give\n appears not only in the traceback, but also in the (short) error message,\n #1572\n - borg.key: include chunk id in exception msgs, #1571\n - better messages for cache newer than repo, #1700\n- vagrant (testing/build VMs):\n\n - upgrade OSXfuse / FUSE for macOS to 3.5.2\n - update Debian Wheezy boxes, #1686\n - openbsd / netbsd: use own boxes, fixes misc rsync installation and\n FUSE/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728\n- docs:\n\n - add docs for \"key export\" and \"key import\" commands, #1641\n - fix inconsistency in FAQ (pv-wrapper).\n - fix second block in \"Easy to use\" section not showing on GitHub, #1576\n - add bestpractices badge\n - link reference docs and faq about BORG_FILES_CACHE_TTL, #1561\n - improve borg info --help, explain size infos, #1532\n - add release signing key / security contact to README, #1560\n - add contribution guidelines for developers\n - development.rst: add sphinx_rtd_theme to the sphinx install command\n - adjust border color in borg.css\n - add debug-info usage help file\n - internals.rst: fix typos\n - setup.py: fix build_usage to always process all commands\n - added docs explaining multiple --restrict-to-path flags, #1602\n - add more specific warning about write-access debug commands, #1587\n - clarify FAQ regarding backup of virtual machines, #1672\n- tests:\n\n - work around FUSE xattr test issue with recent fakeroot\n - simplify repo/hashindex tests\n - travis: test FUSE-enabled borg, use trusty to have a recent FUSE\n - re-enable FUSE tests for RemoteArchiver (no deadlocks any more)\n - clean env for pytest based tests, #1714\n - fuse_mount contextmanager: accept any options\n\n\nVersion 1.0.7 (2016-08-19)\n--------------------------\n\nSecurity fixes:\n\n- borg serve: fix security issue with remote repository access, #1428\n If you used e.g. --restrict-to-path /path/client1/ (with or without trailing\n slash does not make a difference), it acted like a path prefix match using\n /path/client1 (note the missing trailing slash) - the code then also allowed\n working in e.g. /path/client13 or /path/client1000.\n\n As this could accidentally lead to major security/privacy issues depending on\n the paths you use, the behaviour was changed to be a strict directory match.\n That means --restrict-to-path /path/client1 (with or without trailing slash\n does not make a difference) now uses /path/client1/ internally (note the\n trailing slash here!) for matching and allows precisely that path AND any\n path below it. So, /path/client1 is allowed, /path/client1/repo1 is allowed,\n but not /path/client13 or /path/client1000.\n\n If you willingly used the undocumented (dangerous) previous behaviour, you\n may need to rearrange your --restrict-to-path paths now. We are sorry if\n that causes work for you, but we did not want a potentially dangerous\n behaviour in the software (not even using a for-backwards-compat option).\n\nBug fixes:\n\n- fixed repeated LockTimeout exceptions when borg serve tried to write into\n a already write-locked repo (e.g. by a borg mount), #502 part b)\n This was solved by the fix for #1220 in 1.0.7rc1 already.\n- fix cosmetics + file leftover for \"not a valid borg repository\", #1490\n- Cache: release lock if cache is invalid, #1501\n- borg extract --strip-components: fix leak of preloaded chunk contents\n- Repository, when a InvalidRepository exception happens:\n\n - fix spurious, empty lock.roster\n - fix repo not closed cleanly\n\nNew features:\n\n- implement borg debug-info, fixes #1122\n (just calls already existing code via cli, same output as below tracebacks)\n\nOther changes:\n\n- skip the O_NOATIME test on GNU Hurd, fixes #1315\n (this is a very minor issue and the GNU Hurd project knows the bug)\n- document using a clean repo to test / build the release\n\n\nVersion 1.0.7rc2 (2016-08-13)\n-----------------------------\n\nBug fixes:\n\n- do not write objects to repository that are bigger than the allowed size,\n borg will reject reading them, #1451.\n\n Important: if you created archives with many millions of files or\n directories, please verify if you can open them successfully,\n e.g. try a \"borg list REPO::ARCHIVE\".\n- lz4 compression: dynamically enlarge the (de)compression buffer, the static\n buffer was not big enough for archives with extremely many items, #1453\n- larger item metadata stream chunks, raise archive item limit by 8x, #1452\n- fix untracked segments made by moved DELETEs, #1442\n\n Impact: Previously (metadata) segments could become untracked when deleting data,\n these would never be cleaned up.\n- extended attributes (xattrs) related fixes:\n\n - fixed a race condition in xattrs querying that led to the entire file not\n being backed up (while logging the error, exit code = 1), #1469\n - fixed a race condition in xattrs querying that led to a crash, #1462\n - raise OSError including the error message derived from errno, deal with\n path being a integer FD\n\nOther changes:\n\n- print active env var override by default, #1467\n- xattr module: refactor code, deduplicate, clean up\n- repository: split object size check into too small and too big\n- add a transaction_id assertion, so borg init on a broken (inconsistent)\n filesystem does not look like a coding error in borg, but points to the\n real problem.\n- explain confusing TypeError caused by compat support for old servers, #1456\n- add forgotten usage help file from build_usage\n- refactor/unify buffer code into helpers.Buffer class, add tests\n- docs:\n\n - document archive limitation, #1452\n - improve prune examples\n\n\nVersion 1.0.7rc1 (2016-08-05)\n-----------------------------\n\nBug fixes:\n\n- fix repo lock deadlocks (related to lock upgrade), #1220\n- catch unpacker exceptions, resync, #1351\n- fix borg break-lock ignoring BORG_REPO env var, #1324\n- files cache performance fixes (fixes unnecessary re-reading/chunking/\n hashing of unmodified files for some use cases):\n\n - fix unintended file cache eviction, #1430\n - implement BORG_FILES_CACHE_TTL, update FAQ, raise default TTL from 10\n to 20, #1338\n- FUSE:\n\n - cache partially read data chunks (performance), #965, #966\n - always create a root dir, #1125\n- use an OrderedDict for helptext, making the build reproducible, #1346\n- RemoteRepository init: always call close on exceptions, #1370 (cosmetic)\n- ignore stdout/stderr broken pipe errors (cosmetic), #1116\n\nNew features:\n\n- better borg versions management support (useful esp. for borg servers\n wanting to offer multiple borg versions and for clients wanting to choose\n a specific server borg version), #1392:\n\n - add BORG_VERSION environment variable before executing \"borg serve\" via ssh\n - add new placeholder {borgversion}\n - substitute placeholders in --remote-path\n\n- borg init --append-only option (makes using the more secure append-only mode\n more convenient. when used remotely, this requires 1.0.7+ also on the borg\n server), #1291.\n\nOther changes:\n\n- Vagrantfile:\n\n - darwin64: upgrade to FUSE for macOS 3.4.1 (aka osxfuse), #1378\n - xenial64: use user \"ubuntu\", not \"vagrant\" (as usual), #1331\n- tests:\n\n - fix FUSE tests on OS X, #1433\n- docs:\n\n - FAQ: add backup using stable filesystem names recommendation\n - FAQ about glibc compatibility added, #491, glibc-check improved\n - FAQ: 'A' unchanged file; remove ambiguous entry age sentence.\n - OS X: install pkg-config to build with FUSE support, fixes #1400\n - add notes about shell/sudo pitfalls with env. vars, #1380\n - added platform feature matrix\n- implement borg debug-dump-repo-objs\n\n\nVersion 1.0.6 (2016-07-12)\n--------------------------\n\nBug fixes:\n\n- Linux: handle multiple LD_PRELOAD entries correctly, #1314, #1111\n- Fix crash with unclear message if the libc is not found, #1314, #1111\n\nOther changes:\n\n- tests:\n\n - Fixed O_NOATIME tests for Solaris and GNU Hurd, #1315\n - Fixed sparse file tests for (file) systems not supporting it, #1310\n- docs:\n\n - Fixed syntax highlighting, #1313\n - misc docs: added data processing overview picture\n\n\nVersion 1.0.6rc1 (2016-07-10)\n-----------------------------\n\nNew features:\n\n- borg check --repair: heal damaged files if missing chunks re-appear (e.g. if\n the previously missing chunk was added again in a later backup archive),\n #148. (*) Also improved logging.\n\nBug fixes:\n\n- sync_dir: silence fsync() failing with EINVAL, #1287\n Some network filesystems (like smbfs) don't support this and we use this in\n repository code.\n- borg mount (FUSE):\n\n - fix directories being shadowed when contained paths were also specified,\n #1295\n - raise I/O Error (EIO) on damaged files (unless -o allow_damaged_files is\n used), #1302. (*)\n- borg extract: warn if a damaged file is extracted, #1299. (*)\n- Added some missing return code checks (ChunkIndex._add, hashindex_resize).\n- borg check: fix/optimize initial hash table size, avoids resize of the table.\n\nOther changes:\n\n- tests:\n\n - add more FUSE tests, #1284\n - deduplicate FUSE (u)mount code\n - fix borg binary test issues, #862\n- docs:\n\n - changelog: added release dates to older borg releases\n - fix some sphinx (docs generator) warnings, #881\n\nNotes:\n\n(*) Some features depend on information (chunks_healthy list) added to item\nmetadata when a file with missing chunks was \"repaired\" using all-zero\nreplacement chunks. The chunks_healthy list is generated since borg 1.0.4,\nthus borg can't recognize such \"repaired\" (but content-damaged) files if the\nrepair was done with an older borg version.\n\n\nVersion 1.0.5 (2016-07-07)\n--------------------------\n\nBug fixes:\n\n- borg mount: fix FUSE crash in xattr code on Linux introduced in 1.0.4, #1282\n\nOther changes:\n\n- backport some FAQ entries from master branch\n- add release helper scripts\n- Vagrantfile:\n\n - centos6: no FUSE, don't build binary\n - add xz for redhat-like dists\n\n\nVersion 1.0.4 (2016-07-07)\n--------------------------\n\nNew features:\n\n- borg serve --append-only, #1168\n This was included because it was a simple change (append-only functionality\n was already present via repository config file) and makes better security now\n practically usable.\n- BORG_REMOTE_PATH environment variable, #1258\n This was included because it was a simple change (--remote-path cli option\n was already present) and makes borg much easier to use if you need it.\n- Repository: cleanup incomplete transaction on \"no space left\" condition.\n In many cases, this can avoid a 100% full repo filesystem (which is very\n problematic as borg always needs free space - even to delete archives).\n\nBug fixes:\n\n- Fix wrong handling and reporting of OSErrors in borg create, #1138.\n This was a serious issue: in the context of \"borg create\", errors like\n repository I/O errors (e.g. disk I/O errors, ssh repo connection errors)\n were handled badly and did not lead to a crash (which would be good for this\n case, because the repo transaction would be incomplete and trigger a\n transaction rollback to clean up).\n Now, error handling for source files is cleanly separated from every other\n error handling, so only problematic input files are logged and skipped.\n- Implement fail-safe error handling for borg extract.\n Note that this isn't nearly as critical as the borg create error handling\n bug, since nothing is written to the repo. So this was \"merely\" misleading\n error reporting.\n- Add missing error handler in directory attr restore loop.\n- repo: make sure write data hits disk before the commit tag (#1236) and also\n sync the containing directory.\n- FUSE: getxattr fail must use errno.ENOATTR, #1126\n (fixes Mac OS X Finder malfunction: \"zero bytes\" file length, access denied)\n- borg check --repair: do not lose information about the good/original chunks.\n If we do not lose the original chunk IDs list when \"repairing\" a file\n (replacing missing chunks with all-zero chunks), we have a chance to \"heal\"\n the file back into its original state later, in case the chunks re-appear\n (e.g. in a fresh backup). Healing is not implemented yet, see #148.\n- fixes for --read-special mode:\n\n - ignore known files cache, #1241\n - fake regular file mode, #1214\n - improve symlinks handling, #1215\n- remove passphrase from subprocess environment, #1105\n- Ignore empty index file (will trigger index rebuild), #1195\n- add missing placeholder support for --prefix, #1027\n- improve exception handling for placeholder replacement\n- catch and format exceptions in arg parsing\n- helpers: fix \"undefined name 'e'\" in exception handler\n- better error handling for missing repo manifest, #1043\n- borg delete:\n\n - make it possible to delete a repo without manifest\n - borg delete --forced allows one to delete corrupted archives, #1139\n- borg check:\n\n - make borg check work for empty repo\n - fix resync and msgpacked item qualifier, #1135\n - rebuild_manifest: fix crash if 'name' or 'time' key were missing.\n - better validation of item metadata dicts, #1130\n - better validation of archive metadata dicts\n- close the repo on exit - even if rollback did not work, #1197.\n This is rather cosmetic, it avoids repo closing in the destructor.\n\n- tests:\n\n - fix sparse file test, #1170\n - flake8: ignore new F405, #1185\n - catch \"invalid argument\" on cygwin, #257\n - fix sparseness assertion in test prep, #1264\n\nOther changes:\n\n- make borg build/work on OpenSSL 1.0 and 1.1, #1187\n- docs / help:\n\n - fix / clarify prune help, #1143\n - fix \"patterns\" help formatting\n - add missing docs / help about placeholders\n - resources: rename atticmatic to borgmatic\n - document sshd settings, #545\n - more details about checkpoints, add split trick, #1171\n - support docs: add freenode web chat link, #1175\n - add prune visualization / example, #723\n - add note that Fnmatch is default, #1247\n - make clear that lzma levels > 6 are a waste of cpu cycles\n - add a \"do not edit\" note to auto-generated files, #1250\n - update cygwin installation docs\n- repository interoperability with borg master (1.1dev) branch:\n\n - borg check: read item metadata keys from manifest, #1147\n - read v2 hints files, #1235\n - fix hints file \"unknown version\" error handling bug\n- tests: add tests for format_line\n- llfuse: update version requirement for freebsd\n- Vagrantfile:\n\n - use openbsd 5.9, #716\n - do not install llfuse on netbsd (broken)\n - update OSXfuse to version 3.3.3\n - use Python 3.5.2 to build the binaries\n- glibc compatibility checker: scripts/glibc_check.py\n- add .eggs to .gitignore\n\n\nVersion 1.0.3 (2016-05-20)\n--------------------------\n\nBug fixes:\n\n- prune: avoid that checkpoints are kept and completed archives are deleted in\n a prune run), #997\n- prune: fix commandline argument validation - some valid command lines were\n considered invalid (annoying, but harmless), #942\n- fix capabilities extraction on Linux (set xattrs last, after chown()), #1069\n- repository: fix commit tags being seen in data\n- when probing key files, do binary reads. avoids crash when non-borg binary\n files are located in borg's key files directory.\n- handle SIGTERM and make a clean exit - avoids orphan lock files.\n- repository cache: don't cache large objects (avoid using lots of temp. disk\n space), #1063\n\nOther changes:\n\n- Vagrantfile: OS X: update osxfuse / install lzma package, #933\n- setup.py: add check for platform_darwin.c\n- setup.py: on freebsd, use a llfuse release that builds ok\n- docs / help:\n\n - update readthedocs URLs, #991\n - add missing docs for \"borg break-lock\", #992\n - borg create help: add some words to about the archive name\n - borg create help: document format tags, #894\n\n\nVersion 1.0.2 (2016-04-16)\n--------------------------\n\nBug fixes:\n\n- fix malfunction and potential corruption on (nowadays rather rare) big-endian\n architectures or bi-endian archs in (rare) BE mode. #886, #889\n\n cache resync / index merge was malfunctioning due to this, potentially\n leading to data loss. borg info had cosmetic issues (displayed wrong values).\n\n note: all (widespread) little-endian archs (like x86/x64) or bi-endian archs\n in (widespread) LE mode (like ARMEL, MIPSEL, ...) were NOT affected.\n- add overflow and range checks for 1st (special) uint32 of the hashindex\n values, switch from int32 to uint32.\n- fix so that refcount will never overflow, but just stick to max. value after\n a overflow would have occurred.\n- borg delete: fix --cache-only for broken caches, #874\n\n Makes --cache-only idempotent: it won't fail if the cache is already deleted.\n- fixed borg create --one-file-system erroneously traversing into other\n filesystems (if starting fs device number was 0), #873\n- workaround a bug in Linux fadvise FADV_DONTNEED, #907\n\nOther changes:\n\n- better test coverage for hashindex, incl. overflow testing, checking correct\n computations so endianness issues would be discovered.\n- reproducible doc for ProgressIndicator*, make the build reproducible.\n- use latest llfuse for vagrant machines\n- docs:\n\n - use /path/to/repo in examples, fixes #901\n - fix confusing usage of \"repo\" as archive name (use \"arch\")\n\n\nVersion 1.0.1 (2016-04-08)\n--------------------------\n\nNew features:\n\nUsually there are no new features in a bugfix release, but these were added\ndue to their high impact on security/safety/speed or because they are fixes\nalso:\n\n- append-only mode for repositories, #809, #36 (see docs)\n- borg create: add --ignore-inode option to make borg detect unmodified files\n even if your filesystem does not have stable inode numbers (like sshfs and\n possibly CIFS).\n- add options --warning, --error, --critical for missing log levels, #826.\n it's not recommended to suppress warnings or errors, but the user may decide\n this on his own.\n note: --warning is not given to borg serve so a <= 1.0.0 borg will still\n work as server (it is not needed as it is the default).\n do not use --error or --critical when using a <= 1.0.0 borg server.\n\nBug fixes:\n\n- fix silently skipping EIO, #748\n- add context manager for Repository (avoid orphan repository locks), #285\n- do not sleep for >60s while waiting for lock, #773\n- unpack file stats before passing to FUSE\n- fix build on illumos\n- don't try to back up doors or event ports (Solaris and derivatives)\n- remove useless/misleading libc version display, #738\n- test suite: reset exit code of persistent archiver, #844\n- RemoteRepository: clean up pipe if remote open() fails\n- Remote: don't print tracebacks for Error exceptions handled downstream, #792\n- if BORG_PASSPHRASE is present but wrong, don't prompt for password, but fail\n instead, #791\n- ArchiveChecker: move \"orphaned objects check skipped\" to INFO log level, #826\n- fix capitalization, add ellipses, change log level to debug for 2 messages,\n #798\n\nOther changes:\n\n- update llfuse requirement, llfuse 1.0 works\n- update OS / dist packages on build machines, #717\n- prefer showing --info over -v in usage help, #859\n- docs:\n\n - fix cygwin requirements (gcc-g++)\n - document how to debug / file filesystem issues, #664\n - fix reproducible build of api docs\n - RTD theme: CSS !important overwrite, #727\n - Document logo font. Recreate logo png. Remove GIMP logo file.\n\n\nVersion 1.0.0 (2016-03-05)\n--------------------------\n\nThe major release number change (0.x -> 1.x) indicates bigger incompatible\nchanges, please read the compatibility notes, adapt / test your scripts and\ncheck your backup logs.\n\nCompatibility notes:\n\n- drop support for python 3.2 and 3.3, require 3.4 or 3.5, #221 #65 #490\n note: we provide binaries that include python 3.5.1 and everything else\n needed. they are an option in case you are stuck with < 3.4 otherwise.\n- change encryption to be on by default (using \"repokey\" mode)\n- moved keyfile keys from ~/.borg/keys to ~/.config/borg/keys,\n you can either move them manually or run \"borg upgrade \"\n- remove support for --encryption=passphrase,\n use borg migrate-to-repokey to switch to repokey mode, #97\n- remove deprecated --compression ,\n use --compression zlib, instead\n in case of 0, you could also use --compression none\n- remove deprecated --hourly/daily/weekly/monthly/yearly\n use --keep-hourly/daily/weekly/monthly/yearly instead\n- remove deprecated --do-not-cross-mountpoints,\n use --one-file-system instead\n- disambiguate -p option, #563:\n\n - -p now is same as --progress\n - -P now is same as --prefix\n- remove deprecated \"borg verify\",\n use \"borg extract --dry-run\" instead\n- cleanup environment variable semantics, #355\n the environment variables used to be \"yes sayers\" when set, this was\n conceptually generalized to \"automatic answerers\" and they just give their\n value as answer (as if you typed in that value when being asked).\n See the \"usage\" / \"Environment Variables\" section of the docs for details.\n- change the builtin default for --chunker-params, create 2MiB chunks, #343\n --chunker-params new default: 19,23,21,4095 - old default: 10,23,16,4095\n\n one of the biggest issues with borg < 1.0 (and also attic) was that it had a\n default target chunk size of 64kiB, thus it created a lot of chunks and thus\n also a huge chunk management overhead (high RAM and disk usage).\n\n please note that the new default won't change the chunks that you already\n have in your repository. the new big chunks do not deduplicate with the old\n small chunks, so expect your repo to grow at least by the size of every\n changed file and in the worst case (e.g. if your files cache was lost / is\n not used) by the size of every file (minus any compression you might use).\n\n in case you want to see a much lower resource usage immediately (RAM / disk)\n for chunks management, it might be better to start with a new repo than\n to continue in the existing repo (with an existing repo, you have to wait\n until all archives with small chunks get pruned to see a lower resource\n usage).\n\n if you used the old --chunker-params default value (or if you did not use\n --chunker-params option at all) and you'd like to continue using small\n chunks (and you accept the huge resource usage that comes with that), just\n use explicitly borg create --chunker-params=10,23,16,4095.\n- archive timestamps: the 'time' timestamp now refers to archive creation\n start time (was: end time), the new 'time_end' timestamp refers to archive\n creation end time. This might affect prune if your backups take a long time.\n if you give a timestamp via cli, this is stored into 'time'. therefore it now\n needs to mean archive creation start time.\n\nNew features:\n\n- implement password roundtrip, #695\n\nBug fixes:\n\n- remote end does not need cache nor keys directories, do not create them, #701\n- added retry counter for passwords, #703\n\nOther changes:\n\n- fix compiler warnings, #697\n- docs:\n\n - update README.rst to new changelog location in docs/changes.rst\n - add Teemu to AUTHORS\n - changes.rst: fix old chunker params, #698\n - FAQ: how to limit bandwidth\n\n\nVersion 1.0.0rc2 (2016-02-28)\n-----------------------------\n\nNew features:\n\n- format options for location: user, pid, fqdn, hostname, now, utcnow, user\n- borg list --list-format\n- borg prune -v --list enables the keep/prune list output, #658\n\nBug fixes:\n\n- fix _open_rb noatime handling, #657\n- add a simple archivename validator, #680\n- borg create --stats: show timestamps in localtime, use same labels/formatting\n as borg info, #651\n- llfuse compatibility fixes (now compatible with: 0.40, 0.41, 0.42)\n\nOther changes:\n\n- it is now possible to use \"pip install borgbackup[fuse]\" to\n install the llfuse dependency automatically, using the correct version requirement\n for it. you still need to care about having installed the FUSE / build\n related OS package first, though, so that building llfuse can succeed.\n- Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4\n- Vagrant: use pyinstaller v3.1.1 to build binaries\n- docs:\n\n - borg upgrade: add to docs that only LOCAL repos are supported\n - borg upgrade also handles borg 0.xx -> 1.0\n - use pip extras or requirements file to install llfuse\n - fix order in release process\n - updated usage docs and other minor / cosmetic fixes\n - verified borg examples in docs, #644\n - freebsd dependency installation and FUSE configuration, #649\n - add example how to restore a raw device, #671\n - add a hint about the dev headers needed when installing from source\n - add examples for delete (and handle delete after list, before prune), #656\n - update example for borg create -v --stats (use iso datetime format), #663\n - added example to BORG_RSH docs\n - \"connection closed by remote\": add FAQ entry and point to issue #636\n\n\nVersion 1.0.0rc1 (2016-02-07)\n-----------------------------\n\nNew features:\n\n- borg migrate-to-repokey (\"passphrase\" -> \"repokey\" encryption key mode)\n- implement --short for borg list REPO, #611\n- implement --list for borg extract (consistency with borg create)\n- borg serve: overwrite client's --restrict-to-path with ssh forced command's\n option value (but keep everything else from the client commandline), #544\n- use $XDG_CONFIG_HOME/keys for keyfile keys (~/.config/borg/keys), #515\n- \"borg upgrade\" moves the keyfile keys to the new location\n- display both archive creation start and end time in \"borg info\", #627\n\n\nBug fixes:\n\n- normalize trailing slashes for the repository path, #606\n- Cache: fix exception handling in __init__, release lock, #610\n\nOther changes:\n\n- suppress unneeded exception context (PEP 409), simpler tracebacks\n- removed special code needed to deal with imperfections / incompatibilities /\n missing stuff in py 3.2/3.3, simplify code that can be done simpler in 3.4\n- removed some version requirements that were kept on old versions because\n newer did not support py 3.2 any more\n- use some py 3.4+ stdlib code instead of own/openssl/pypi code:\n\n - use os.urandom instead of own cython openssl RAND_bytes wrapper, #493\n - use hashlib.pbkdf2_hmac from py stdlib instead of own openssl wrapper\n - use hmac.compare_digest instead of == operator (constant time comparison)\n - use stat.filemode instead of homegrown code\n - use \"mock\" library from stdlib, #145\n - remove borg.support (with non-broken argparse copy), it is ok in 3.4+, #358\n- Vagrant: copy CHANGES.rst as symlink, #592\n- cosmetic code cleanups, add flake8 to tox/travis, #4\n- docs / help:\n\n - make \"borg -h\" output prettier, #591\n - slightly rephrase prune help\n - add missing example for --list option of borg create\n - quote exclude line that includes an asterisk to prevent shell expansion\n - fix dead link to license\n - delete Ubuntu Vivid, it is not supported anymore (EOL)\n - OS X binary does not work for older OS X releases, #629\n - borg serve's special support for forced/original ssh commands, #544\n - misc. updates and fixes\n" }, { "path": "docs/conf.py", "content": "# Documentation build configuration file, created by\n# sphinx-quickstart on Sat Sep 10 18:18:25 2011.\n#\n# This file is execfile()d with the current directory set to its containing directory.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nimport sys\nimport os\n\nsys.path.insert(0, os.path.abspath(\"../src\"))\n\nfrom borg import __version__ as sw_version\n\n# -- General configuration -----------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be extensions\n# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.\nextensions = []\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix of source filenames.\nsource_suffix = \".rst\"\n\n# The encoding of source files.\n# source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = \"Borg - Deduplicating Archiver\"\ncopyright = \"2010-2014 Jonas Borgström, 2015-2025 The Borg Collective (see AUTHORS file)\"\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nsplit_char = \"+\" if \"+\" in sw_version else \"-\"\nversion = sw_version.split(split_char)[0]\n# The full version, including alpha/beta/rc tags.\nrelease = version\n\nsuppress_warnings = [\"image.nonlocal_uri\"]\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n# language = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n# today = ''\n# Else, today_fmt is used as the format for a strftime call.\ntoday_fmt = \"%Y-%m-%d\"\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = [\"_build\"]\n\n# The reST default role (used for this markup: `text`) to use for all documents.\n# default_role = None\n\n# The Borg docs contain no or very little Python docs.\n# Thus, the primary domain is RST.\nprimary_domain = \"rst\"\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n\n# -- Options for HTML output ---------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of built-in themes.\nimport guzzle_sphinx_theme\n\nhtml_theme_path = guzzle_sphinx_theme.html_theme_path()\nhtml_theme = \"guzzle_sphinx_theme\"\n\n\ndef set_rst_settings(app):\n app.env.settings.update({\"field_name_limit\": 0, \"option_limit\": 0})\n\n\ndef setup(app):\n app.setup_extension(\"sphinxcontrib.jquery\")\n app.add_css_file(\"css/borg.css\")\n app.connect(\"builder-inited\", set_rst_settings)\n\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\nhtml_theme_options = {\"project_nav_name\": \"Borg %s\" % version}\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = ['_themes']\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n# html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\nhtml_logo = \"_static/logo.svg\"\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\nhtml_favicon = \"_static/favicon.ico\"\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = [\"borg_theme\"]\n\nhtml_extra_path = [\"../src/borg/paperkey.html\"]\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\nhtml_last_updated_fmt = \"%Y-%m-%d\"\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\nhtml_use_smartypants = True\nsmartquotes_action = \"qe\" # no D in there means \"do not transform -- and ---\"\n\n# Custom sidebar templates, maps document names to template names.\nhtml_sidebars = {\"**\": [\"logo-text.html\", \"versionselector.html\", \"searchbox.html\", \"globaltoc.html\"]}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n# html_domain_indices = True\n\n# If false, no index is generated.\nhtml_use_index = False\n\n# If true, the index is split into individual pages for each letter.\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\nhtml_show_sourcelink = False\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\nhtml_show_sphinx = False\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\nhtml_show_copyright = False\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"borgdoc\"\n\n\n# -- Options for LaTeX output --------------------------------------------------\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title, author, documentclass [howto/manual]).\nlatex_documents = [(\"book\", \"Borg.tex\", \"Borg Documentation\", \"The Borg Collective\", \"manual\")]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\nlatex_logo = \"_static/logo.pdf\"\n\nlatex_elements = {\"papersize\": \"a4paper\", \"pointsize\": \"10pt\", \"figure_align\": \"H\"}\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\nlatex_show_urls = \"footnote\"\n\n# Additional stuff for the LaTeX preamble.\n# latex_preamble = ''\n\n# Documents to append as an appendix to all manuals.\nlatex_appendices = [\"support\", \"resources\", \"changes\", \"authors\"]\n\n# If false, no module index is generated.\n# latex_domain_indices = True\n\n\n# -- Options for manual page output --------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (\n \"usage\",\n \"borg\",\n \"BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.\",\n [\"The Borg Collective (see AUTHORS file)\"],\n 1,\n )\n]\n\nextensions = [\n \"sphinx.ext.extlinks\",\n \"sphinx.ext.autodoc\",\n \"sphinx.ext.todo\",\n \"sphinx.ext.coverage\",\n \"sphinx.ext.viewcode\",\n \"sphinxcontrib.jquery\", # jquery is not included anymore by default\n \"guzzle_sphinx_theme\", # register the theme as an extension to generate a sitemap.xml\n]\n\nextlinks = {\"issue\": (\"https://github.com/borgbackup/borg/issues/%s\", \"#%s\")}\n" }, { "path": "docs/deployment/automated-local.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n\nAutomated backups to a local hard drive\n=======================================\n\nThis guide shows how to automate backups to a hard drive directly connected\nto your computer. If a backup hard drive is connected, backups are automatically\nstarted, and the drive shut-down and disconnected when they are done.\n\nThis guide is written for a Linux-based operating system and makes use of\nsystemd and udev.\n\nOverview\n--------\n\nA udev rule is created to trigger on the addition of block devices. The rule contains a tag\nthat triggers systemd to start a one-shot service. The one-shot service executes a script in\nthe standard systemd service environment, which automatically captures stdout/stderr and\nlogs it to the journal.\n\nThe script mounts the added block device if it is a registered backup drive and creates\nbackups on it. When done, it optionally unmounts the filesystem and spins the drive down,\nso that it may be physically disconnected.\n\nConfiguring the system\n----------------------\n\nFirst, create the ``/etc/backups`` directory (as root).\nAll configuration goes into this directory.\n\nFind out the ID of the partition table of your backup disk (here assumed to be /dev/sdz)::\n\n lsblk --fs -o +PTUUID /dev/sdz\n\nThen, create ``/etc/backups/80-backup.rules`` with the following content (all on one line)::\n\n ACTION==\"add\", SUBSYSTEM==\"block\", ENV{ID_PART_TABLE_UUID}==\"\", TAG+=\"systemd\", ENV{SYSTEMD_WANTS}+=\"automatic-backup.service\"\n\nThe \"systemd\" tag in conjunction with the SYSTEMD_WANTS environment variable has systemd\nlaunch the \"automatic-backup\" service, which we will create next, as the\n``/etc/backups/automatic-backup.service`` file:\n\n.. code-block:: ini\n\n [Service]\n Type=oneshot\n ExecStart=/etc/backups/run.sh\n\nNow, create the main backup script, ``/etc/backups/run.sh``. Below is a template;\nmodify it to suit your needs (e.g., more backup sets, dumping databases, etc.).\n\n.. code-block:: bash\n\n #!/bin/bash -ue\n\n # The udev rule is not terribly accurate and may trigger our service before\n # the kernel has finished probing partitions. Sleep for a bit to ensure\n # the kernel is done.\n #\n # This can be avoided by using a more precise udev rule, e.g. matching\n # a specific hardware path and partition.\n sleep 5\n\n #\n # Script configuration\n #\n\n # The backup partition is mounted there\n MOUNTPOINT=/mnt/backup\n\n # This is the location of the Borg repository\n TARGET=$MOUNTPOINT/borg-backups/backup.borg\n\n # Archive name schema\n DATE=$(date --iso-8601)-$(hostname)\n\n # This is the file that will later contain UUIDs of registered backup drives\n DISKS=/etc/backups/backup.disks\n\n # Find whether the connected block device is a backup drive\n for uuid in $(lsblk --noheadings --list --output uuid)\n do\n if grep --quiet --fixed-strings $uuid $DISKS; then\n break\n fi\n uuid=\n done\n\n if [ ! $uuid ]; then\n echo \"No backup disk found, exiting\"\n exit 0\n fi\n\n echo \"Disk $uuid is a backup disk\"\n partition_path=/dev/disk/by-uuid/$uuid\n # Mount filesystem if not already done. This assumes that if something is already\n # mounted at $MOUNTPOINT, it is the backup drive. It will not find the drive if\n # it was mounted somewhere else.\n findmnt $MOUNTPOINT >/dev/null || mount $partition_path $MOUNTPOINT\n drive=$(lsblk --inverse --noheadings --list --paths --output name $partition_path | head --lines 1)\n echo \"Drive path: $drive\"\n\n #\n # Create backups\n #\n\n # Options for borg create\n BORG_OPTS=\"--stats --one-file-system --compression lz4\"\n\n # Set BORG_PASSPHRASE or BORG_PASSCOMMAND somewhere around here, using export,\n # if encryption is used.\n\n # Because no one can answer these questions non-interactively, it is better to\n # fail quickly instead of hanging.\n export BORG_RELOCATED_REPO_ACCESS_IS_OK=no\n export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no\n\n # Log Borg version\n borg --version\n\n echo \"Starting backup for $DATE\"\n\n # This is just an example, change it however you see fit\n borg create $BORG_OPTS \\\n --exclude root/.cache \\\n --exclude var/lib/docker/devicemapper \\\n $TARGET::$DATE-$$-system \\\n / /boot\n\n # /home is often a separate partition/filesystem.\n # Even if it is not (add --exclude /home above), it probably makes sense\n # to have /home in a separate archive.\n borg create $BORG_OPTS \\\n --exclude 'sh:home/*/.cache' \\\n $TARGET::$DATE-$$-home \\\n /home/\n\n echo \"Completed backup for $DATE\"\n\n # Just to be completely paranoid\n sync\n\n if [ -f /etc/backups/autoeject ]; then\n umount $MOUNTPOINT\n hdparm -Y $drive\n fi\n\n if [ -f /etc/backups/backup-suspend ]; then\n systemctl suspend\n fi\n\nCreate the ``/etc/backups/autoeject`` file to have the script automatically eject the drive\nafter creating the backup. Rename the file to something else (e.g., ``/etc/backups/autoeject-no``)\nwhen you want to do something with the drive after creating backups (e.g., running checks).\n\nCreate the ``/etc/backups/backup-suspend`` file if the machine should suspend after completing\nthe backup. Don't forget to disconnect the device physically before resuming,\notherwise you'll enter a cycle. You can also add an option to power down instead.\n\nCreate an empty ``/etc/backups/backup.disks`` file, in which you will register your backup drives.\n\nFinally, enable the udev rules and services:\n\n.. code-block:: bash\n\n ln -s /etc/backups/80-backup.rules /etc/udev/rules.d/80-backup.rules\n ln -s /etc/backups/automatic-backup.service /etc/systemd/system/automatic-backup.service\n systemctl daemon-reload\n udevadm control --reload\n\nAdding backup hard drives\n-------------------------\n\nConnect your backup hard drive. Format it, if not done already.\nFind the UUID of the filesystem on which backups should be stored::\n\n lsblk -o+uuid,label\n\nRecord the UUID in the ``/etc/backups/backup.disks`` file.\n\nMount the drive at /mnt/backup.\n\nInitialize a Borg repository at the location indicated by ``TARGET``::\n\n borg init --encryption ... /mnt/backup/borg-backups/backup.borg\n\nUnmount and reconnect the drive, or manually start the ``automatic-backup`` service\nto start the first backup::\n\n systemctl start --no-block automatic-backup\n\nSee backup logs using journalctl::\n\n journalctl -fu automatic-backup [-n number-of-lines]\n\nSecurity considerations\n-----------------------\n\nThe script as shown above will mount any filesystem with a UUID listed in\n``/etc/backups/backup.disks``. The UUID check is a safety/annoyance-reduction\nmechanism to keep the script from blowing up whenever a random USB thumb drive is connected.\nIt is not meant as a security mechanism. Mounting filesystems and reading repository\ndata exposes additional attack surfaces (kernel filesystem drivers,\npossibly userspace services, and Borg itself). On the other hand, someone\nstanding right next to your computer can attempt a lot of attacks, most of which\nare easier to do than, e.g., exploiting filesystems (installing a physical keylogger,\nDMA attacks, stealing the machine, ...).\n\nBorg ensures that backups are not created on random drives that \"just happen\"\nto contain a Borg repository. If an unknown unencrypted repository is encountered,\nthen the script aborts (BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no).\n\nBackups are only created on hard drives that contain a Borg repository that is\neither known (by ID) to your machine or you are using encryption and the\npassphrase of the repository has to match the passphrase supplied to Borg.\n" }, { "path": "docs/deployment/central-backup-server.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n.. _central-backup-server:\n\nCentral repository server with Ansible or Salt\n==============================================\n\nThis section gives an example of how to set up a Borg repository server for multiple\nclients.\n\nMachines\n--------\n\nThis section uses multiple machines, referred to by their\nrespective fully qualified domain names (FQDNs).\n\n* The backup server: `backup01.srv.local`\n* The clients:\n\n - John Doe's desktop: `johndoe.clnt.local`\n - Web server 01: `web01.srv.local`\n - Application server 01: `app01.srv.local`\n\nUser and group\n--------------\n\nThe repository server should have a single UNIX user for all the clients.\nRecommended user and group with additional settings:\n\n* User: `backup`\n* Group: `backup`\n* Shell: `/bin/bash` (or another shell capable of running the `borg serve` command)\n* Home: `/home/backup`\n\nMost clients should initiate a backup as the root user to capture all\nusers, groups, and permissions (e.g., when backing up `/home`).\n\nFolders\n-------\n\nThe following directory layout is suggested on the repository server:\n\n* User home directory, /home/backup\n* Repositories path (storage pool): /home/backup/repos\n* Clients’ restricted paths (`/home/backup/repos/`):\n\n - johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local`\n - web01.srv.local: `/home/backup/repos/web01.srv.local`\n - app01.srv.local: `/home/backup/repos/app01.srv.local`\n\nRestrictions\n------------\n\nBorg is instructed to restrict clients into their own paths:\n``borg serve --restrict-to-path /home/backup/repos/``\n\nThe client will be able to access any file or subdirectory inside of ``/home/backup/repos/``\nbut no other directories. You can allow a client to access several separate directories by passing multiple\n``--restrict-to-path`` flags, for instance: ``borg serve --restrict-to-path /home/backup/repos/ --restrict-to-path /home/backup/repos/``,\nwhich could make sense if multiple machines belong to one person which should then have access to all the\nbackups of their machines.\n\nOnly one SSH key per client is allowed. Keys are added for ``johndoe.clnt.local``, ``web01.srv.local`` and\n``app01.srv.local``. They will access the backup under a single UNIX user account as\n``backup@backup01.srv.local``. Every key in ``$HOME/.ssh/authorized_keys`` has a\nforced command and restrictions applied, as shown below:\n\n::\n\n command=\"cd /home/backup/repos/;\n borg serve --restrict-to-path /home/backup/repos/\",\n restrict \n\n.. note:: The text shown above needs to be written on a single line!\n\nThe options added to the key perform the following:\n\n1. Change working directory\n2. Run ``borg serve`` restricted to the client base path\n3. Restrict ssh and do not allow stuff which imposes a security risk\n\nBecause of the ``cd`` command, the server automatically changes the current\nworking directory. The client then does not need to know the absolute\nor relative remote repository path and can directly access the repositories at\n``ssh://@/./``.\n\n.. note:: The setup above ignores all client-given command line parameters\n that are normally appended to the `borg serve` command.\n\nClient\n------\n\nThe client needs to initialize the `pictures` repository like this:\n\n::\n\n borg init ssh://backup@backup01.srv.local/./pictures\n\nOr with the full path (this should not be used in practice; it is only for demonstration purposes).\nThe server automatically changes the current working directory to the `` directory.\n\n::\n\n borg init ssh://backup@backup01.srv.local/home/backup/repos/johndoe.clnt.local/pictures\n\nWhen `johndoe.clnt.local` tries to access a path outside its restriction, the following error is raised.\nJohn Doe tries to back up into the web01 path:\n\n::\n\n borg init ssh://backup@backup01.srv.local/home/backup/repos/web01.srv.local/pictures\n\n::\n\n ~~~ SNIP ~~~\n Remote: borg.remote.PathNotAllowed: /home/backup/repos/web01.srv.local/pictures\n ~~~ SNIP ~~~\n Repository path not allowed\n\nAnsible\n-------\n\nAnsible takes care of all the system-specific commands to add the user, create the\nfolder, install and configure software.\n\n::\n\n - hosts: backup01.srv.local\n vars:\n user: backup\n group: backup\n home: /home/backup\n pool: \"{{ home }}/repos\"\n auth_users:\n - host: johndoe.clnt.local\n key: \"{{ lookup('file', '/path/to/keys/johndoe.clnt.local.pub') }}\"\n - host: web01.clnt.local\n key: \"{{ lookup('file', '/path/to/keys/web01.clnt.local.pub') }}\"\n - host: app01.clnt.local\n key: \"{{ lookup('file', '/path/to/keys/app01.clnt.local.pub') }}\"\n tasks:\n - package: name=borg state=present\n - group: name=\"{{ group }}\" state=present\n - user: name=\"{{ user }}\" shell=/bin/bash home=\"{{ home }}\" createhome=yes group=\"{{ group }}\" groups= state=present\n - file: path=\"{{ home }}\" owner=\"{{ user }}\" group=\"{{ group }}\" mode=0700 state=directory\n - file: path=\"{{ home }}/.ssh\" owner=\"{{ user }}\" group=\"{{ group }}\" mode=0700 state=directory\n - file: path=\"{{ pool }}\" owner=\"{{ user }}\" group=\"{{ group }}\" mode=0700 state=directory\n - authorized_key: user=\"{{ user }}\"\n key=\"{{ item.key }}\"\n key_options='command=\"cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}\",restrict'\n with_items: \"{{ auth_users }}\"\n - file: path=\"{{ home }}/.ssh/authorized_keys\" owner=\"{{ user }}\" group=\"{{ group }}\" mode=0600 state=file\n - file: path=\"{{ pool }}/{{ item.host }}\" owner=\"{{ user }}\" group=\"{{ group }}\" mode=0700 state=directory\n with_items: \"{{ auth_users }}\"\n\nSalt\n----\n\nThis is a configuration similar to the one above, configured to be deployed with\nSalt running on a Debian system.\n\n::\n\n Install borg backup from pip:\n pkg.installed:\n - pkgs:\n - python3\n - python3-dev\n - python3-pip\n - python-virtualenv\n - libssl-dev\n - openssl\n - libacl1-dev\n - libacl1\n - build-essential\n - libfuse-dev\n - fuse\n - pkg-config\n pip.installed:\n - pkgs: [\"borgbackup\"]\n - bin_env: /usr/bin/pip3\n\n Setup backup user:\n user.present:\n - name: backup\n - fullname: Backup User\n - home: /home/backup\n - shell: /bin/bash\n # CAUTION!\n # If you change the ssh command= option below, it won't necessarily get pushed to the backup\n # server correctly unless you delete the ~/.ssh/authorized_keys file and re-create it!\n {% for host in backupclients %}\n Give backup access to {{host}}:\n ssh_auth.present:\n - user: backup\n - source: salt://conf/ssh-pubkeys/{{host}}-backup.id_ecdsa.pub\n - options:\n - command=\"cd /home/backup/repos/{{host}}; borg serve --restrict-to-path /home/backup/repos/{{host}}\"\n - restrict\n {% endfor %}\n\n\nEnhancements\n------------\n\nAs this section only describes a simple and effective setup, it could be further\nenhanced when supporting (a limited set) of client supplied commands. A wrapper\nfor starting `borg serve` could be written. Or borg itself could be enhanced to\nautodetect it runs under SSH by checking the `SSH_ORIGINAL_COMMAND` environment\nvariable. This is left open for future improvements.\n\nWhen extending ssh autodetection in borg no external wrapper script is necessary\nand no other interpreter or application has to be deployed.\n\nSee also\n--------\n\n* `SSH Daemon manpage `_\n* `Ansible `_\n* `Salt `_\n" }, { "path": "docs/deployment/hosting-repositories.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n.. _hosting_repositories:\n\nHosting repositories\n====================\n\nThis section shows how to provide repository storage securely for users.\n\nRepositories are accessed through SSH. Each user of the service should\nhave their own login, which is only able to access that user's files.\nTechnically, it is possible to have multiple users share one login;\nhowever, separating them is better. Separate logins increase isolation\nand provide an additional layer of security and safety for both the\nprovider and the users.\n\nFor example, if a user manages to breach ``borg serve``, they can\nonly damage their own data (assuming that the system does not have further\nvulnerabilities).\n\nUse the standard directory structure of the operating system. Each user\nis assigned a home directory, and that user's repositories reside in their\nhome directory.\n\nThe following ``~user/.ssh/authorized_keys`` file is the most important\npiece for a correct deployment. It allows the user to log in via\ntheir public key (which must be provided by the user), and restricts\nSSH access to safe operations only.\n\n::\n\n command=\"borg serve --restrict-to-repository /home//repository\",restrict\n \n\n.. note:: The text shown above needs to be written on a **single** line!\n\n.. warning::\n\n If this file should be automatically updated (e.g. by a web console),\n pay **utmost attention** to sanitizing user input. Strip all whitespace\n around the user-supplied key, ensure that it **only** contains ASCII\n with no control characters and that it consists of three parts separated\n by a single space. Ensure that no newlines are contained within the key.\n\nThe ``restrict`` keyword enables all restrictions, i.e. disables port, agent\nand X11 forwarding, as well as disabling PTY allocation and execution of ~/.ssh/rc.\nIf any future restriction capabilities are added to authorized_keys\nfiles they will be included in this set.\n\nThe ``command`` keyword forces execution of the specified command\nupon login. This must be ``borg serve``. The ``--restrict-to-repository``\noption permits access to exactly **one** repository. It can be given\nmultiple times to permit access to more than one repository.\n\nThe repository may not exist yet; it can be initialized by the user,\nwhich allows for encryption.\n\nRefer to the `sshd(8) `_\nman page for more details on SSH options.\nSee also :ref:`borg_serve`\n" }, { "path": "docs/deployment/image-backup.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n\nBacking up entire disk images\n=============================\n\nBacking up disk images can still be efficient with Borg because its `deduplication`_\ntechnique makes sure only the modified parts of the file are stored. Borg also has\noptional simple sparse file support for extraction.\n\nIt is of utmost importance to pin down the disk you want to back up.\nUse the disk's SERIAL for that.\nUse:\n\n.. code-block:: bash\n\n # You can find the short disk serial by:\n # udevadm info --query=property --name=nvme1n1 | grep ID_SERIAL_SHORT | cut -d '=' -f 2\n export BORG_REPO=/path/to/repo\n DISK_SERIAL=\"7VS0224F\"\n DISK_ID=$(readlink -f /dev/disk/by-id/*\"${DISK_SERIAL}\") # Returns /dev/nvme1n1\n\n mapfile -t PARTITIONS < <(lsblk -o NAME,TYPE -p -n -l \"$DISK_ID\" | awk '$2 == \"part\" {print $1}')\n echo \"Partitions of $DISK_ID:\"\n echo \"${PARTITIONS[@]}\"\n echo \"Disk Identifier: $DISK_ID\"\n\n # Use the following line to perform a Borg backup for the full disk:\n # borg create --read-special disk-backup \"$DISK_ID\"\n\n # Use the following to perform a Borg backup for all partitions of the disk\n # borg create --read-special partitions-backup \"${PARTITIONS[@]}\"\n\n # Example output:\n # Partitions of /dev/nvme1n1:\n # /dev/nvme1n1p1\n # /dev/nvme1n1p2\n # /dev/nvme1n1p3\n # Disk Identifier: /dev/nvme1n1\n # borg create --read-special disk-backup /dev/nvme1n1\n # borg create --read-special partitions-backup /dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3\n\n\n\n\nDecreasing the size of image backups\n------------------------------------\n\nDisk images are as large as the full disk when uncompressed and might not get much\nsmaller post-deduplication after heavy use because virtually all filesystems do not\nactually delete file data on disk but instead delete the filesystem entries referencing\nthe data. Therefore, if a disk nears capacity and files are deleted again, the change\nwill barely decrease the space it takes up when compressed and deduplicated. Depending\non the filesystem, there are several ways to decrease the size of a disk image:\n\nUsing ntfsclone (NTFS, i.e. Windows VMs)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``ntfsclone`` can only operate on filesystems with the journal cleared (i.e. turned-off\nmachines), which somewhat limits its utility in the case of VM snapshots. However, when\nit can be used, its special image format is even more efficient than just zeroing and\ndeduplicating. For backup, save the disk header and the contents of each partition::\n\n HEADER_SIZE=$(sfdisk -lo Start $DISK | grep -A1 -P 'Start$' | tail -n1 | xargs echo)\n PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\\s*Type/d')\n dd if=$DISK count=$HEADER_SIZE | borg create --repo repo hostname-partinfo -\n echo \"$PARTITIONS\" | grep NTFS | cut -d' ' -f1 | while read x; do\n PARTNUM=$(echo $x | grep -Eo \"[0-9]+$\")\n ntfsclone -so - $x | borg create --repo repo hostname-part$PARTNUM -\n done\n # to back up non-NTFS partitions as well:\n echo \"$PARTITIONS\" | grep -v NTFS | cut -d' ' -f1 | while read x; do\n PARTNUM=$(echo $x | grep -Eo \"[0-9]+$\")\n borg create --read-special --repo repo hostname-part$PARTNUM $x\n done\n\nRestoration is a similar process::\n\n borg extract --stdout --repo repo hostname-partinfo | dd of=$DISK && partprobe\n PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\\s*Type/d')\n borg list --format {archive}{NL} repo | grep 'part[0-9]*$' | while read x; do\n PARTNUM=$(echo $x | grep -Eo \"[0-9]+$\")\n PARTITION=$(echo \"$PARTITIONS\" | grep -E \"$DISKp?$PARTNUM\" | head -n1)\n if echo \"$PARTITION\" | cut -d' ' -f2- | grep -q NTFS; then\n borg extract --stdout --repo repo $x | ntfsclone -rO $(echo \"$PARTITION\" | cut -d' ' -f1) -\n else\n borg extract --stdout --repo repo $x | dd of=$(echo \"$PARTITION\" | cut -d' ' -f1)\n fi\n done\n\n.. note::\n\n When backing up a disk image (as opposed to a real block device), mount it as\n a loopback image to use the above snippets::\n\n DISK=$(losetup -Pf --show /path/to/disk/image)\n # do backup as shown above\n losetup -d $DISK\n\nUsing zerofree (ext2, ext3, ext4)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``zerofree`` works similarly to ntfsclone in that it zeros out unused chunks of the FS,\nexcept it works in place, zeroing the original partition. This makes the backup process\na bit simpler::\n\n sfdisk -lo Device,Type $DISK | sed -e '1,/Device\\s*Type/d' | grep Linux | cut -d' ' -f1 | xargs -n1 zerofree\n borg create --read-special --repo repo hostname-disk $DISK\n\nBecause the partitions were zeroed in place, restoration is only one command::\n\n borg extract --stdout --repo repo hostname-disk | dd of=$DISK\n\n.. note:: The \"traditional\" way to zero out space on a partition, especially one already\n mounted, is simply to ``dd`` from ``/dev/zero`` to a temporary file and delete\n it. This is ill-advised for the reasons mentioned in the ``zerofree`` man page:\n\n - it is slow.\n - it makes the disk image (temporarily) grow to its maximal extent.\n - it (temporarily) uses all free space on the disk, so other concurrent write actions may fail.\n\nVirtual machines\n----------------\n\nIf you use non-snapshotting backup tools like Borg to back up virtual machines, then\nthe VMs should be turned off for the duration of the backup. Backing up live VMs can\n(and will) result in corrupted or inconsistent backup contents: a VM image is just a\nregular file to Borg with the same issues as regular files when it comes to concurrent\nreading and writing from the same file.\n\nFor backing up live VMs use filesystem snapshots on the VM host, which establishes\ncrash-consistency for the VM images. This means that with most filesystems (that\nare journaling) the FS will always be fine in the backup (but may need a journal\nreplay to become accessible).\n\nUsually this does not mean that file *contents* on the VM are consistent, since file\ncontents are normally not journaled. Notable exceptions are ext4 in data=journal mode,\nZFS and btrfs (unless nodatacow is used).\n\nApplications designed with crash-consistency in mind (most relational databases like\nPostgreSQL, SQLite etc. but also for example Borg repositories) should always be able\nto recover to a consistent state from a backup created with crash-consistent snapshots\n(even on ext4 with data=writeback or XFS). Other applications may require a lot of work\nto reach application-consistency; it's a broad and complex issue that cannot be explained\nin entirety here.\n\nHypervisor snapshots capturing most of the VM's state can also be used for backups and\ncan be a better alternative to pure filesystem-based snapshots of the VM's disk, since\nno state is lost. Depending on the application this can be the easiest and most reliable\nway to create application-consistent backups.\n\nBorg does not intend to address these issues due to their huge complexity and\nplatform/software dependency. Combining Borg with the mechanisms provided by the platform\n(snapshots, hypervisor features) will be the best approach to start tackling them.\n" }, { "path": "docs/deployment/non-root-user.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n.. _non_root_user:\n\n================================\nBacking up using a non-root user\n================================\n\nThis section describes how to run Borg as a non-root user and still be able to\nback up every file on the system.\n\nNormally, Borg is run as the root user to bypass all filesystem permissions and\nbe able to read all files. However, in theory this also allows Borg to modify or\ndelete files on your system (for example, in case of a bug).\n\nTo eliminate this possibility, we can run Borg as a non-root user and give it read-only\npermissions to all files on the system.\n\n\nUsing Linux capabilities inside a systemd service\n=================================================\n\nOne way to do so is to use Linux `capabilities\n`_ within a systemd\nservice.\n\nLinux capabilities allow us to grant parts of the root user’s privileges to\na non-root user. This works on a per-thread level and does not grant permissions\nto the non-root user as a whole.\n\nFor this, we need to run the backup script from a systemd service and use the `AmbientCapabilities\n`_\noption added in systemd 229.\n\nA very basic unit file would look like this:\n\n::\n\n [Unit]\n Description=Borg Backup\n\n [Service]\n Type=oneshot\n User=borg\n ExecStart=/usr/local/sbin/backup.sh\n\n AmbientCapabilities=CAP_DAC_READ_SEARCH\n\nThe ``CAP_DAC_READ_SEARCH`` capability gives Borg read-only access to all files and directories on the system.\n\nThis service can then be started manually using ``systemctl start``, a systemd timer or other methods.\n\nRestore considerations\n======================\n\nUse the root user when restoring files. If you use the non-root user, ``borg extract`` will\nchange ownership of all restored files to the non-root user. Using ``borg mount`` will not allow the\nnon-root user to access files it would not be able to access on the system itself.\n\nOther than that, you can use the same restore process you would use when running the backup as root.\n\n.. warning::\n\n When using a local repository and running Borg commands as root, make sure to use only commands that do not\n modify the repository itself, such as extract or mount. Modifying the repository as root will break it for the\n non-root user, since some files inside the repository will then be owned by root.\n" }, { "path": "docs/deployment/pull-backup.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n.. _pull_backup:\n\n=======================\nBacking up in pull mode\n=======================\n\nTypically the Borg client connects to a backup server using SSH as a transport\nwhen initiating a backup. This is referred to as push mode.\n\nHowever, if you require the backup server to initiate the connection, or prefer\nit to initiate the backup run, one of the following workarounds is required to\nallow such a pull-mode setup.\n\nA common use case for pull mode is to back up a remote server to a local personal\ncomputer.\n\nSSHFS\n=====\n\nAssume you have a pull backup system set up with Borg, where a backup server\npulls data from the target via SSHFS. In this mode, the backup client's filesystem\nis mounted remotely on the backup server. Pull mode is even possible if\nthe SSH connection must be established by the client via a remote tunnel. Other\nnetwork file systems like NFS or SMB could be used as well, but SSHFS is very\nsimple to set up and probably the most secure one.\n\nThere are some restrictions caused by SSHFS. For example, unless you define UID\nand GID mappings when mounting via ``sshfs``, owners and groups of the mounted\nfilesystem will probably change, and you may not have access to those files if\nBorg is not run with root privileges.\n\nSSHFS is a FUSE filesystem and uses the SFTP protocol, so there may also be\nunsupported features that the actual implementations of SSHFS, libfuse, and\nSFTP on the backup server do not support, like filename encodings, ACLs, xattrs,\nor flags. Therefore, there is no guarantee that you can restore a system\ncompletely in every aspect from such a backup.\n\n.. warning::\n\n To mount the client's root filesystem you will need root access to the\n client. This contradicts the usual threat model of Borg, where\n clients do not need to trust the backup server (data is encrypted). In pull\n mode the server (when logged in as root) could cause unlimited damage to the\n client. Therefore, pull mode should be used only with servers you fully\n trust!\n\n.. warning::\n\n Additionally, while chrooted into the client's root filesystem,\n code from the client will be executed. Therefore, you should do this only when\n you fully trust the client.\n\n.. warning::\n\n The chroot method was chosen to get the right user and group name-id\n mappings, assuming they only come from files (/etc/passwd and group).\n This assumption might be wrong, e.g. if users/groups also come from\n ldap or other providers.\n Thus, it might be better to use ``--numeric-ids`` and not archive any\n user or group names (but just the numeric IDs) and not use chroot.\n\nCreating a backup\n-----------------\n\nGenerally, in a pull backup situation there is no direct way for borg to know\nthe client's original UID:GID name mapping of files, because Borg would use\n``/etc/passwd`` and ``/etc/group`` of the backup server to map the names. To\nderive the right names, Borg needs to have access to the client's passwd and\ngroup files and use them in the backup process.\n\nThe solution to this problem is chrooting into an sshfs mounted directory. In\nthis example the whole client root file system is mounted. We use the\nstand-alone BorgBackup executable and copy it into the mounted file system to\nmake Borg available after entering chroot; this can be skipped if Borg is\nalready installed on the client.\n\n::\n\n # Mount client root file system.\n mkdir /tmp/sshfs\n sshfs root@host:/ /tmp/sshfs\n # Mount BorgBackup repository inside it.\n mkdir /tmp/sshfs/borgrepo\n mount --bind /path/to/repo /tmp/sshfs/borgrepo\n # Make borg executable available.\n cp /usr/local/bin/borg /tmp/sshfs/usr/local/bin/borg\n # Mount important system directories and enter chroot.\n cd /tmp/sshfs\n for i in dev proc sys; do mount --bind /$i $i; done\n chroot /tmp/sshfs\n\nNow we are on the backup system but inside a chroot with the client's root file\nsystem. We have a copy of Borg binary in ``/usr/local/bin`` and the repository\nin ``/borgrepo``. Borg will back up the client's user/group names, and we can\ncreate the backup, retaining the original paths, excluding the repository:\n\n::\n\n borg create --exclude borgrepo --files-cache ctime,size --repo /borgrepo archive /\n\nFor the sake of simplicity only ``borgrepo`` is excluded here. You may want to\nset up an exclude file with additional files and folders to be excluded. Also\nnote that we have to modify Borg's file change detection behaviour – SSHFS\ncannot guarantee stable inode numbers, so we have to supply the\n``--files-cache`` option.\n\nFinally, we need to exit chroot, unmount all the stuff and clean up:\n\n::\n\n exit # exit chroot\n rm /tmp/sshfs/usr/local/bin/borg\n cd /tmp/sshfs\n for i in dev proc sys borgrepo; do umount ./$i; done\n rmdir borgrepo\n cd ~\n umount /tmp/sshfs\n rmdir /tmp/sshfs\n\nThanks to secuser on IRC for this how-to!\n\nRestore methods\n---------------\n\nThe counterpart of a pull backup is a push restore. Depending on the type of\nrestore – full restore or partial restore – there are different methods to make\nsure the correct IDs are restored.\n\nPartial restore\n~~~~~~~~~~~~~~~\n\nIn case of a partial restore, using the archived UIDs/GIDs might lead to wrong\nresults if the name-to-ID mapping on the target system has changed compared to\nbackup time (might be the case e.g. for a fresh OS install).\n\nThe workaround again is chrooting into an sshfs mounted directory, so Borg is\nable to map the user/group names of the backup files to the actual IDs on the\nclient. This example is similar to the backup above – only the Borg command is\ndifferent:\n\n::\n\n # Mount client root file system.\n mkdir /tmp/sshfs\n sshfs root@host:/ /tmp/sshfs\n # Mount BorgBackup repository inside it.\n mkdir /tmp/sshfs/borgrepo\n mount --bind /path/to/repo /tmp/sshfs/borgrepo\n # Make borg executable available.\n cp /usr/local/bin/borg /tmp/sshfs/usr/local/bin/borg\n # Mount important system directories and enter chroot.\n cd /tmp/sshfs\n for i in dev proc sys; do mount --bind /$i $i; done\n chroot /tmp/sshfs\n\nNow we can run\n\n::\n\n borg extract --repo /borgrepo archive PATH\n\nto restore whatever we like partially. Finally, do the clean-up:\n\n::\n\n exit # exit chroot\n rm /tmp/sshfs/usr/local/bin/borg\n cd /tmp/sshfs\n for i in dev proc sys borgrepo; do umount ./$i; done\n rmdir borgrepo\n cd ~\n umount /tmp/sshfs\n rmdir /tmp/sshfs\n\nFull restore\n~~~~~~~~~~~~\n\nWhen doing a full restore, we restore all files (including the ones containing\nthe ID-to-name mapping, ``/etc/passwd`` and ``/etc/group``). Everything will be\nconsistent automatically if we restore the numeric IDs stored in the archive. So\nthere is no need for a chroot environment; we just mount the client file system\nand extract a backup, utilizing the ``--numeric-ids`` option:\n\n::\n\n sshfs root@host:/ /mnt/sshfs\n cd /mnt/sshfs\n borg extract --numeric-ids --repo /path/to/repo archive\n cd ~\n umount /mnt/sshfs\n\nSimple (lossy) full restore\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nUsing ``borg export-tar`` it is possible to stream a backup to the client and\ndirectly extract it without the need of mounting with SSHFS:\n\n::\n\n borg export-tar --repo /path/to/repo archive - | ssh root@host 'tar -C / -x'\n\nNote that in this scenario the tar format is the limiting factor – it cannot\nrestore all the advanced features that BorgBackup supports. See\n:ref:`borg_export-tar` for limitations.\n\nsocat\n=====\n\nIn this setup a SSH connection from the backup server to the client is\nestablished that uses SSH reverse port forwarding to tunnel data\ntransparently between UNIX domain sockets on the client and server and the socat\ntool to connect these with the borg client and server processes, respectively.\n\nThe program socat has to be available on the backup server and on the client\nto be backed up.\n\nWhen **pushing** a backup the borg client (holding the data to be backed up)\nconnects to the backup server via ssh, starts ``borg serve`` on the backup\nserver and communicates via standard input and output (transported via SSH)\nwith the process on the backup server.\n\nWith the help of socat this process can be reversed. The backup server will\ncreate a connection to the client (holding the data to be backed up) and will\n**pull** the data.\n\nIn the following example *borg-server* connects to *borg-client* to pull a backup.\n\nTo provide a secure setup sockets should be stored in ``/run/borg``, only\naccessible to the users that run the backup process. So on both systems,\n*borg-server* and *borg-client* the folder ``/run/borg`` has to be created::\n\n sudo mkdir -m 0700 /run/borg\n\nOn *borg-server* the socket file is opened by the user running the ``borg\nserve`` process writing to the repository\nso the user has to have read and write permissions on ``/run/borg``::\n\n borg-server:~$ sudo chown borgs /run/borg\n\nOn *borg-client* the socket file is created by ssh, so the user used to connect\nto *borg-client* has to have read and write permissions on ``/run/borg``::\n\n borg-client:~$ sudo chown borgc /run/borg\n\nOn *borg-server*, we have to start the command ``borg serve`` and make its\nstandard input and output available to a unix socket::\n\n borg-server:~$ socat UNIX-LISTEN:/run/borg/reponame.sock,fork EXEC:\"borg serve --restrict-to-path /path/to/repo\"\n\nSocat will wait until a connection is opened. Then socat will execute the\ncommand given, redirecting Standard Input and Output to the unix socket. The\noptional arguments for ``borg serve`` are not necessary but a sane default.\n\n.. note::\n When used in production you may also use systemd socket-based activation\n instead of socat on the server side. You would wrap the ``borg serve`` command\n in a `service unit`_ and configure a matching `socket unit`_\n to start the service whenever a client connects to the socket.\n\n .. _service unit: https://www.freedesktop.org/software/systemd/man/systemd.service.html\n .. _socket unit: https://www.freedesktop.org/software/systemd/man/systemd.socket.html\n\nNow we need a way to access the unix socket on *borg-client* (holding the\ndata to be backed up), as we created the unix socket on *borg-server*\nOpening a SSH connection from the *borg-server* to the *borg-client* with reverse port\nforwarding can do this for us::\n\n borg-server:~$ ssh -R /run/borg/reponame.sock:/run/borg/reponame.sock borgc@borg-client\n\n.. note::\n\n As the default value of OpenSSH for ``StreamLocalBindUnlink`` is ``no``, the\n socket file created by sshd is not removed. Trying to connect a second time,\n will print a short warning, and the forwarding does **not** take place::\n\n Warning: remote port forwarding failed for listen path /run/borg/reponame.sock\n\n When you are done, you have to remove the socket file manually, otherwise\n you may see an error like this when trying to execute borg commands::\n\n Remote: YYYY/MM/DD HH:MM:SS socat[XXX] E connect(5, AF=1 \"/run/borg/reponame.sock\", 13): Connection refused\n Connection closed by remote host. Is borg working on the server?\n\n\nWhen a process opens the socket on *borg-client*, SSH will forward all\ndata to the socket on *borg-server*.\n\nThe next step is to tell borg on *borg-client* to use the unix socket to communicate with the\n``borg serve`` command on *borg-server* via the socat socket instead of SSH::\n\n borg-client:~$ export BORG_RSH=\"sh -c 'exec socat STDIO UNIX-CONNECT:/run/borg/reponame.sock'\"\n\nThe default value for ``BORG_RSH`` is ``ssh``. By default Borg uses SSH to create\nthe connection to the backup server. Therefore Borg parses the repo URL\nand adds the server name (and other arguments) to the SSH command. Those\narguments can not be handled by socat. We wrap the command with ``sh`` to\nignore all arguments intended for the SSH command.\n\nAll Borg commands can now be executed on *borg-client*. For example to create a\nbackup execute the ``borg create`` command::\n\n borg-client:~$ borg create --repo ssh://borg-server/path/to/repo archive /path_to_backup\n\nWhen automating backup creation, the\ninteractive ssh session may seem inappropriate. An alternative way of creating\na backup may be the following command::\n\n borg-server:~$ ssh \\\n -R /run/borg/reponame.sock:/run/borg/reponame.sock \\\n borgc@borg-client \\\n borg create \\\n --rsh \"sh -c 'exec socat STDIO UNIX-CONNECT:/run/borg/reponame.sock'\" \\\n --repo ssh://borg-server/path/to/repo archive /path_to_backup \\\n ';' rm /run/borg/reponame.sock\n\nThis command also automatically removes the socket file after the ``borg\ncreate`` command is done.\n\nssh-agent\n=========\n\nIn this scenario *borg-server* initiates an SSH connection to *borg-client* and forwards the authentication\nagent connection.\n\nAfter that, it works similar to the push mode:\n*borg-client* initiates another SSH connection back to *borg-server* using the forwarded authentication agent\nconnection to authenticate itself, starts ``borg serve`` and communicates with it.\n\nUsing this method requires ssh access of user *borgs* to *borgc@borg-client*, where:\n\n* *borgs* is the user on the server side with read/write access to local borg repository.\n* *borgc* is the user on the client side with read access to files meant to be backed up.\n\nApplying this method for automated backup operations\n----------------------------------------------------\n\nAssume that the borg-client host is untrusted.\nTherefore we do some effort to prevent a hostile user on the borg-client side to do something harmful.\nIn case of a fully trusted borg-client the method could be simplified.\n\nPreparing the server side\n~~~~~~~~~~~~~~~~~~~~~~~~~\n\nDo this once for each client on *borg-server* to allow *borgs* to connect itself on *borg-server* using a\ndedicated ssh key:\n\n::\n\n borgs@borg-server$ install -m 700 -d ~/.ssh/\n borgs@borg-server$ ssh-keygen -N '' -t rsa -f ~/.ssh/borg-client_key\n borgs@borg-server$ { echo -n 'command=\"borg serve --restrict-to-repo ~/repo\",restrict '; cat ~/.ssh/borg-client_key.pub; } >> ~/.ssh/authorized_keys\n borgs@borg-server$ chmod 600 ~/.ssh/authorized_keys\n\n``install -m 700 -d ~/.ssh/``\n\n Create directory ~/.ssh with correct permissions if it does not exist yet.\n\n``ssh-keygen -N '' -t rsa -f ~/.ssh/borg-client_key``\n\n Create an ssh key dedicated to communication with borg-client.\n\n.. note::\n Another more complex approach is using a unique ssh key for each pull operation.\n This is more secure as it guarantees that the key will not be used for other purposes.\n\n``{ echo -n 'command=\"borg serve --restrict-to-repo ~/repo\",restrict '; cat ~/.ssh/borg-client_key.pub; } >> ~/.ssh/authorized_keys``\n\n Add borg-client's ssh public key to ~/.ssh/authorized_keys with forced command and restricted mode.\n The borg client is restricted to use one repo at the specified path.\n\n``chmod 600 ~/.ssh/authorized_keys``\n\n Fix permissions of ~/.ssh/authorized_keys.\n\nPull operation\n~~~~~~~~~~~~~~\n\nInitiating borg command execution from *borg-server* (e.g. init)::\n\n borgs@borg-server$ (\n eval $(ssh-agent) > /dev/null\n ssh-add -q ~/.ssh/borg-client_key\n echo 'your secure borg key passphrase' | \\\n ssh -A -o StrictHostKeyChecking=no borgc@borg-client \"BORG_PASSPHRASE=\\$(cat) borg --rsh 'ssh -o StrictHostKeyChecking=no' init --encryption repokey ssh://borgs@borg-server/~/repo\"\n kill \"${SSH_AGENT_PID}\"\n )\n\nParentheses around commands are needed to avoid interference with a possibly already running ssh-agent.\nParentheses are not needed when using a dedicated bash process.\n\n``eval $(ssh-agent) > /dev/null``\n\n Run the SSH agent in the background and export related environment variables to the current bash session.\n\n``ssh-add -q ~/.ssh/borg-client_key``\n\n Load the SSH private key dedicated to communication with the borg-client into the SSH agent.\n Look at ``man 1 ssh-add`` for a more detailed explanation.\n\n.. note::\n Care needs to be taken when loading keys into the SSH agent. Users on the *borg-client* having read/write permissions\n to the agent's UNIX-domain socket (at least borgc and root in our case) can access the agent on *borg-server* through\n the forwarded connection and can authenticate using any of the identities loaded into the agent\n (look at ``man 1 ssh`` for more detailed explanation). Therefore there are some security considerations:\n\n * Private keys loaded into the agent must not be used to enable access anywhere else.\n * The keys meant to be loaded into the agent must be specified explicitly, not from default locations.\n * The *borg-client*'s entry in *borgs@borg-server:~/.ssh/authorized_keys* must be as restrictive as possible.\n\n``echo 'your secure borg key passphrase' | ssh -A -o StrictHostKeyChecking=no borgc@borg-client \"BORG_PASSPHRASE=\\$(cat) borg --rsh 'ssh -o StrictHostKeyChecking=no' init --encryption repokey ssh://borgs@borg-server/~/repo\"``\n\n Run the *borg init* command on *borg-client*.\n\n *ssh://borgs@borg-server/~/repo* refers to the repository *repo* within borgs's home directory on *borg-server*.\n\n *StrictHostKeyChecking=no* is used to add host keys automatically to *~/.ssh/known_hosts* without user intervention.\n\n``kill \"${SSH_AGENT_PID}\"``\n\n Kill ssh-agent with loaded keys when it is not needed anymore.\n\nRemote forwarding\n=================\n\nThe standard ssh client allows to create tunnels to forward local ports to a remote server (local forwarding) and also\nto allow remote ports to be forwarded to local ports (remote forwarding).\n\nThis remote forwarding can be used to allow remote backup clients to access the backup server even if the backup server\ncannot be reached by the backup client.\n\nThis can even be used in cases where neither the backup server can reach the backup client and the backup client cannot\nreach the backup server, but some intermediate host can access both.\n\nA schematic approach is as follows\n\n::\n\n Backup Server (backup@mybackup) Intermediate Machine (john@myinter) Backup Client (bob@myclient)\n\n 1. Establish SSH remote forwarding -----------> SSH listen on local port\n\n 2. Starting ``borg create`` establishes\n 3. SSH forwards to intermediate machine <------- SSH connection to the local port\n 4. Receives backup connection <------- and further on to backup server\n via SSH\n\nSo for the backup client the backup is done via SSH to a local port and for the backup server there is a normal backup\nperformed via ssh.\n\nIn order to achieve this, the following commands can be used to create the remote port forwarding:\n\n1. On machine ``myinter``\n\n``ssh bob@myclient -v -C -R 8022:mybackup:22 -N``\n\nThis will listen for ssh-connections on port ``8022`` on ``myclient`` and forward connections to port 22 on ``mybackup``.\n\nYou can also remove the need for machine ``myinter`` and create the port forwarding on the backup server directly by\nusing ``localhost`` instead of ``mybackup``\n\n2. On machine ``myclient``\n\n``borg create -v --progress --stats ssh://backup@localhost:8022/home/backup/repos/myclient /``\n\nMake sure to use port ``8022`` and ``localhost`` for the repository as this instructs borg on ``myclient`` to use the\nremote forwarded ssh connection.\n\nSSH Keys\n--------\n\nIf you want to automate backups when using this method, the ssh ``known_hosts`` and ``authorized_keys`` need to be set up\nto allow connections.\n\nSecurity Considerations\n-----------------------\n\nOpening up SSH access this way can pose a security risk as it effectively opens remote access to your\nbackup server on the client even if it is located outside of your company network.\n\nTo reduce the chances of compromise, you should configure a forced command in ``authorized_keys`` to prevent\nanyone from performing any other action on the backup server.\n\nThis can be done e.g. by adding the following in ``$HOME/.ssh/authorized_keys`` on ``mybackup`` with proper\npath and client-fqdn:\n\n::\n\n command=\"cd /home/backup/repos/;borg serve --restrict-to-path /home/backup/repos/\"\n\n\nAll the additional security considerations for borg should be applied, see :ref:`central-backup-server` for some additional\nhints.\n\nMore information\n----------------\n\nSee `remote forwarding`_ and the `ssh man page`_ for more information about remote forwarding.\n\n .. _remote forwarding: https://linuxize.com/post/how-to-setup-ssh-tunneling/\n .. _ssh man page: https://manpages.debian.org/testing/manpages-de/ssh.1.de.html\n" }, { "path": "docs/deployment.rst", "content": ".. include:: global.rst.inc\n.. highlight:: none\n\nDeployment\n==========\n\nThis chapter details deployment strategies for the following scenarios.\n\n.. toctree::\n :titlesonly:\n\n deployment/central-backup-server\n deployment/hosting-repositories\n deployment/automated-local\n deployment/image-backup\n deployment/pull-backup\n deployment/non-root-user\n" }, { "path": "docs/development.rst", "content": ".. include:: global.rst.inc\n.. highlight:: bash\n.. _development:\n\nDevelopment\n===========\n\nThis chapter will get you started with Borg development.\n\nBorg is written in Python (with a little bit of Cython and C for\nthe performance-critical parts).\n\nContributions\n-------------\n\n... are welcome!\n\nSome guidance for contributors:\n\n- Discuss changes on the GitHub issue tracker, on IRC or on the mailing list.\n\n- Make your PRs on the ``master`` branch (see `Branching Model`_ for details and exceptions).\n\n- Do clean changesets:\n\n - Focus on some topic, resist changing anything else.\n - Do not do style changes mixed with functional changes.\n - Try to avoid refactorings mixed with functional changes.\n - If you need to fix something after commit/push:\n\n - If there are ongoing reviews: do a fixup commit you can\n squash into the bad commit later.\n - If there are no ongoing reviews or you did not push the\n bad commit yet: amend the commit to include your fix or\n merge the fixup commit before pushing.\n - Have a nice, clear, typo-free commit comment.\n - If you fixed an issue, refer to it in your commit comment.\n - Follow the style guide (see below).\n\n- If you write new code, please add tests and docs for it.\n\n- Run the tests, fix any issues that come up.\n\n- Make a pull request on GitHub.\n\n- Wait for review by other developers.\n\nBranching model\n---------------\n\nBorg development happens on the ``master`` branch and uses GitHub pull\nrequests (if you don't have GitHub or don't want to use it you can\nsend smaller patches via the borgbackup mailing list to the maintainers).\n\nStable releases are maintained on maintenance branches named ``x.y-maint``, e.g.,\nthe maintenance branch of the 1.4.x series is ``1.4-maint``.\n\nMost PRs should be filed against the ``master`` branch. Only if an\nissue affects **only** a particular maintenance branch a PR should be\nfiled against it directly.\n\nWhile discussing/reviewing a PR it will be decided whether the\nchange should be applied to maintenance branches. Each maintenance\nbranch has a corresponding *backport/x.y-maint* label, which will then\nbe applied.\n\nChanges that are typically considered for backporting:\n\n- Data loss, corruption and inaccessibility fixes.\n- Security fixes.\n- Forward-compatibility improvements.\n- Documentation corrections.\n\n.. rubric:: Maintainer part\n\nFrom time to time a maintainer will backport the changes for a\nmaintenance branch, typically before a release or if enough changes\nwere collected:\n\n1. Notify others that you're doing this to avoid duplicate work.\n2. Branch a backporting branch off the maintenance branch.\n3. Cherry pick and backport the changes from each labelled PR, remove\n the label for each PR you've backported.\n\n To preserve authorship metadata, do not follow the ``git cherry-pick``\n instructions to use ``git commit`` after resolving conflicts. Instead,\n stage conflict resolutions and run ``git cherry-pick --continue``,\n much like using ``git rebase``.\n\n To avoid merge issues (a cherry pick is a form of merge), use\n these options (similar to the ``git merge`` options used previously,\n the ``-x`` option adds a reference to the original commit)::\n\n git cherry-pick --strategy recursive -X rename-threshold=5% -x\n\n4. Make a PR of the backporting branch against the maintenance branch\n for backport review. Mention the backported PRs in this PR, e.g.:\n\n Includes changes from #2055 #2057 #2381\n\n This way GitHub will automatically show in these PRs where they\n were backported.\n\n.. rubric:: Historic model\n\nPreviously (until release 1.0.10) Borg used a `\"merge upwards\"\n`_ model where\nmost minor changes and fixes were committed to a maintenance branch\n(e.g. 1.0-maint), and the maintenance branch(es) were regularly merged\nback into the main development branch. This became more and more\ntroublesome due to merges growing more conflict-heavy and error-prone.\n\nHow to submit a pull request\n----------------------------\n\nIn order to contribute to Borg, you will need to fork the ``borgbackup/borg``\nmain repository to your own Github repository. Then clone your Github repository\nto your local machine. The instructions for forking and cloning a repository\ncan be found there:\n``_ .\n\nMake sure you also fetched the git tags, because without them, ``setuptools-scm``\nwill run into issues determining the correct borg version. Check if ``git tag``\nshows a lot of release tags (version numbers).\nIf it does not, use ``git fetch --tags`` to fetch them.\n\nTo work on your contribution, you first need to decide which branch your pull\nrequest should be against. Often, this might be master branch (esp. for big /\nrisky contributions), but it could be also a maintenance branch like e.g.\n1.4-maint (esp. for small fixes that should go into next maintenance release,\ne.g. 1.4.x).\n\nStart by checking out the appropriate branch:\n::\n\n git checkout master\n\nIt is best practice for a developer to keep local ``master`` branch as an\nup-to-date copy of the upstream ``master`` branch and always do own work in a\nseparate feature or bugfix branch.\nThis is useful to be able to rebase own branches onto the upstream branches\nthey were branched from, if necessary.\n\nThis also applies to other upstream branches (like e.g. ``1.4-maint``), not\nonly to ``master``.\n\nThus, create a new branch now:\n::\n\n git checkout -b MYCONTRIB-master # choose an appropriate own branch name\n\nNow, work on your contribution in that branch. Use these git commands:\n::\n\n git status # is there anything that needs to be added?\n git add ... # if so, add it\n git commit # finally, commit it. use a descriptive comment.\n\nThen push the changes to your Github repository:\n::\n\n git push --set-upstream origin MYCONTRIB-master\n\nFinally, make a pull request on ``borgbackup/borg`` Github repository against\nthe appropriate branch (e.g. ``master``) so that your changes can be reviewed.\n\nWhat to do if work was accidentally started in wrong branch\n-----------------------------------------------------------\n\nIf you accidentally worked in ``master`` branch, check out the ``master``\nbranch and make sure there are no uncommitted changes. Then, create a feature\nbranch from that, so that your contribution is in a feature branch.\n::\n\n git checkout master\n git checkout -b MYCONTRIB-master\n\nNext, check out the ``master`` branch again. Find the commit hash of the last\ncommit that was made before you started working on your contribution and perform\na hard reset.\n::\n\n git checkout master\n git log\n git reset --hard THATHASH\n\nThen, update the local ``master`` branch with changes made in the upstream\nrepository.\n::\n\n git pull borg master\n\nRebase feature branch onto updated master branch\n------------------------------------------------\n\nAfter updating the local ``master`` branch from upstream, the feature branch\ncan be checked out and rebased onto (the now up-to-date) ``master`` branch.\n::\n\n git checkout MYCONTRIB-master\n git rebase -i master\n\nNext, check if there are any commits that exist in the feature branch\nbut not in the ``master`` branch and vice versa. If there are no\nconflicts or after resolving them, push your changes to your Github repository.\n::\n\n git log\n git diff master\n git push -f\n\nCode and issues\n---------------\n\nCode is stored on GitHub, in the `Borgbackup organization\n`_. `Issues\n`_ and `pull requests\n`_ should be sent there as\nwell. See also the :ref:`support` section for more details.\n\nStyle guide / Automated Code Formatting\n---------------------------------------\n\nWe use `black`_ for automatically formatting the code.\n\nIf you work on the code, it is recommended that you run black **before each commit**\n(so that new code is always using the desired formatting and no additional commits\nare required to fix the formatting).\n::\n\n pip install -r requirements.d/codestyle.txt # everybody use same black version\n black --check . # only check, don't change\n black . # reformat the code\n\n\nThe CI workflows will check the code formatting and will fail if it is not formatted correctly.\n\nWhen (mass-)reformatting existing code, we need to avoid ruining `git blame`, so please\nfollow their `guide about avoiding ruining git blame`_:\n\n.. _black: https://black.readthedocs.io/\n.. _guide about avoiding ruining git blame: https://black.readthedocs.io/en/stable/guides/introducing_black_to_your_project.html#avoiding-ruining-git-blame\n\nContinuous Integration\n----------------------\n\nAll pull requests go through `GitHub Actions`_, which runs the tests on misc.\nPython versions and on misc. platforms as well as some additional checks.\n\n.. _GitHub Actions: https://github.com/borgbackup/borg/actions\n\nOutput and Logging\n------------------\nWhen writing logger calls, always use correct log level (debug only for\ndebugging, info for informative messages, warning for warnings, error for\nerrors, critical for critical errors/states).\n\nWhen directly talking to the user (e.g. Y/N questions), do not use logging,\nbut directly output to stderr (not: stdout, it could be connected to a pipe).\n\nTo control the amount and kinds of messages output emitted at info level, use\nflags like ``--stats`` or ``--list``, then create a topic logger for messages\ncontrolled by that flag. See ``_setup_implied_logging()`` in\n``borg/archiver.py`` for the entry point to topic logging.\n\nBuilding a development environment\n----------------------------------\n\nFirst, just install borg into a virtual env :ref:`as described before `.\n\nTo install some additional packages needed for running the tests, activate your\nvirtual env and run::\n\n pip install -r requirements.d/development.lock.txt\n\n\nThis project utilizes pre-commit to format and lint code before it is committed.\nAlthough pre-commit is installed when running the command above, the pre-commit hooks\nwill have to be installed separately. Run this command to install the pre-commit hooks::\n\n pre-commit install\n\nRunning the tests\n-----------------\n\nThe tests are in the borg/testsuite package.\n\nTo run all the tests, you need to have fakeroot installed. If you do not have\nfakeroot, you still will be able to run most tests, just leave away the\n``fakeroot -u`` from the given command lines.\n\nTo run the test suite use the following command::\n\n fakeroot -u tox # run all tests\n\nSome more advanced examples::\n\n # verify a changed tox.ini (run this after any change to tox.ini):\n fakeroot -u tox --recreate\n\n fakeroot -u tox -e py313 # run all tests, but only on python 3.13\n\n fakeroot -u tox borg.testsuite.locking # only run 1 test module\n\n fakeroot -u tox borg.testsuite.locking -- -k '\"not Timer\"' # exclude some tests\n\n fakeroot -u tox borg.testsuite -- -v # verbose py.test\n\nImportant notes:\n\n- When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``.\n\nRunning the tests (using the pypi package)\n------------------------------------------\n\nSince borg 1.4, it is also possible to run the tests without a development\nenvironment, using the borgbackup dist package (downloaded from pypi.org or\ngithub releases page):\n::\n\n # optional: create and use a virtual env:\n python3 -m venv env\n . env/bin/activate\n\n # install packages\n pip install borgbackup\n pip install pytest pytest-benchmark\n\n # run the tests\n pytest -v -rs --benchmark-skip --pyargs borg.testsuite\n\nAdding a compression algorithm\n------------------------------\n\nIf you want to add a new compression algorithm, please refer to :issue:`1633`\nand leave a post there in order to discuss about the proposal.\n\nDocumentation\n-------------\n\nGenerated files\n~~~~~~~~~~~~~~~\n\nUsage documentation (found in ``docs/usage/``) and man pages\n(``docs/man/``) are generated automatically from the command line\nparsers declared in the program and their documentation, which is\nembedded in the program (see archiver.py). These are committed to git\nfor easier use by packagers downstream.\n\nWhen a command is added, a command line flag changed, added or removed,\nthe usage docs need to be rebuilt as well::\n\n python scripts/make.py build_usage\n python scripts/make.py build_man\n\nHowever, we prefer to do this as part of our :ref:`releasing`\npreparations, so it is generally not necessary to update these when\nsubmitting patches that change something about the command line.\n\nBuilding the docs with Sphinx\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe documentation (in reStructuredText format, .rst) is in docs/.\n\nTo build the html version of it, you need to have Sphinx installed\n(in your Borg virtualenv with Python 3)::\n\n pip install -r requirements.d/docs.txt\n\nNow run::\n\n cd docs/\n make html\n\nThen point a web browser at docs/_build/html/index.html.\n\nThe website is updated automatically by ReadTheDocs through GitHub web hooks on the\nmain repository.\n\nUsing Vagrant\n-------------\n\nWe use Vagrant for the automated creation of testing environments and borgbackup\nstandalone binaries for various platforms.\n\nFor better security, there is no automatic sync in the VM to host direction.\nThe plugin `vagrant-scp` is useful to copy stuff from the VMs to the host.\n\nThe \"windows10\" box requires the `reload` plugin (``vagrant plugin install vagrant-reload``).\n\nUsage::\n\n # To create and provision the VM:\n vagrant up OS\n # same, but use 6 VM cpus and 12 workers for pytest:\n VMCPUS=6 XDISTN=12 vagrant up OS\n # To create an ssh session to the VM:\n vagrant ssh OS\n # To execute a command via ssh in the VM:\n vagrant ssh OS -c \"command args\"\n # To shut down the VM:\n vagrant halt OS\n # To shut down and destroy the VM:\n vagrant destroy OS\n # To copy files from the VM (in this case, the generated binary):\n vagrant scp OS:/vagrant/borg/borg.exe .\n\nUsing Podman\n------------\n\nmacOS-based developers (and others who prefer containers) can run the Linux test suite locally using Podman.\n\nPrerequisites:\n\n- Install Podman (e.g., ``brew install podman``).\n- Initialize the Podman machine, only once: ``podman machine init``.\n- Start the Podman machine, before using it: ``podman machine start``.\n\nUsage::\n\n # Open an interactive shell in the container (default if no command given):\n ./scripts/linux-run\n\n # Run the default tox environment:\n ./scripts/linux-run tox\n\n # Run a specific tox environment:\n ./scripts/linux-run tox -e py311-pyfuse3\n\n # Pass arguments to pytest (e.g., run specific tests):\n ./scripts/linux-run tox -e py313-pyfuse3 -- -k mount\n\n # Switch base image (temporarily):\n ./scripts/linux-run --image python:3.11-bookworm tox\n\nResource Usage\n~~~~~~~~~~~~~~\n\nThe default Podman VM uses 2GB RAM and half your CPUs.\nFor heavy tests (parallel execution), this might be tight.\n\n- **Check usage:** Run ``podman stats`` in another terminal while tests are running.\n- **Increase resources:**\n\n ::\n\n podman machine stop\n podman machine set --cpus 6 --memory 4096\n podman machine start\n\n\nCreating standalone binaries\n----------------------------\n\nMake sure you have everything built and installed (including fuse stuff).\nWhen using the Vagrant VMs, pyinstaller will already be installed.\n\nWith virtual env activated::\n\n pip install pyinstaller # or git checkout master\n pyinstaller -F -n borg-PLATFORM borg/__main__.py\n for file in dist/borg-*; do gpg --armor --detach-sign $file; done\n\nIf you encounter issues, see also our `Vagrantfile` for details.\n\n.. note:: Standalone binaries built with pyinstaller are supposed to\n work on same OS, same architecture (x86 32bit, amd64 64bit)\n without external dependencies.\n\n.. _releasing:\n\nCreating a new release\n----------------------\n\nChecklist:\n\n- Make sure all issues for this milestone are closed or moved to the\n next milestone.\n- Check if there are any pending fixes for security issues.\n- Find and fix any low hanging fruit left on the issue tracker.\n- Check that GitHub Actions CI is happy.\n- Update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``.\n- Check version number of upcoming release in ``CHANGES.rst``.\n- Render ``CHANGES.rst`` via ``make html`` and check for markup errors.\n- Verify that ``MANIFEST.in``, ``pyproject.toml`` and ``setup.py`` are complete.\n- Run these commands, check git status for files that might need to be added, and commit::\n\n python scripts/make.py build_usage\n python scripts/make.py build_man\n\n- Tag the release::\n\n git tag -s -m \"tagged/signed release X.Y.Z\" X.Y.Z\n\n- Push the release PR branch to GitHub, make a pull request.\n- Also push the release tag.\n- Create a clean repo and use it for the following steps::\n\n git clone borg borg-clean\n\n This makes sure no uncommitted files get into the release archive.\n It will also reveal uncommitted required files.\n Moreover, it makes sure the vagrant machines only get committed files and\n do a fresh start based on that.\n- Optional: run tox and/or binary builds on all supported platforms via vagrant,\n check for test failures. This is now optional as we do platform testing and\n binary building on GitHub.\n- Create sdist, sign it, upload release to (test) PyPi:\n\n ::\n\n scripts/sdist-sign X.Y.Z\n scripts/upload-pypi X.Y.Z test\n scripts/upload-pypi X.Y.Z\n\n Note: the signature is not uploaded to PyPi any more, but we upload it to\n github releases.\n- When GitHub CI looks good on the release PR, merge it and then check \"Actions\":\n GitHub will create binary assets after the release PR is merged within the\n CI testing of the merge. Check the \"Upload binaries\" step on Ubuntu (AMD/Intel\n and ARM64) and macOS (Intel and ARM64), fetch the ZIPs with the binaries.\n- Unpack the ZIPs and test the binaries, upload the binaries to the GitHub\n release page (borg-OS-SPEC-ARCH-gh and borg-OS-SPEC-ARCH-gh.tgz).\n\n- Close the release milestone on GitHub.\n- `Update borgbackup.org\n `_ with the\n new version number and release date.\n- Announce on:\n\n - Mailing list.\n - Mastodon / BlueSky / X (aka Twitter).\n - IRC channel (change ``/topic``).\n\n- Create a GitHub release, include:\n\n - pypi dist package and signature\n - Standalone binaries (see above for how to create them).\n\n - For macOS binaries **with** FUSE support, document the macFUSE version\n in the README of the binaries. macFUSE uses a kernel extension that needs\n to be compatible with the code contained in the binary.\n - A link to ``CHANGES.rst``.\n" }, { "path": "docs/faq.rst", "content": ".. include:: global.rst.inc\n.. highlight:: none\n.. _faq:\n\nFrequently asked questions\n==========================\n\nUsage & Limitations\n###################\n\nWhat is the difference between a repository on an external hard drive and a repository on a server?\n---------------------------------------------------------------------------------------------------\n\nIf Borg is running in client/server mode, the client uses SSH as a transport to\ntalk to a remote agent, which is another Borg process (Borg is also installed on\nthe server) started automatically by the client. The Borg server performs\nstorage-related, low-level repository operations (list, load, and store objects),\nwhile the Borg client does the high-level stuff: deduplication, encryption,\ncompression, dealing with archives, backups, restores, etc., which reduces the\namount of data that goes over the network.\n\nWhen Borg is writing to a repo on a locally mounted remote filesystem, e.g.\nSSHFS, the Borg client can only perform filesystem operations and has no agent\nrunning on the remote side, so *every* operation needs to go over the network,\nwhich is slower.\n\nCan I back up from multiple servers into a single repository?\n-------------------------------------------------------------\n\nYes, you can! Even simultaneously.\n\nCan I back up to multiple swapped backup targets?\n--------------------------------------------------\n\nIt is possible to swap your backup disks if each backup medium is assigned its\nown repository by creating a new one with :ref:`borg_repo-create`.\n\nCan I copy or synchronize my repo to another location?\n------------------------------------------------------\n\nIf you want to have redundant backup repositories (preferably at separate\nlocations), the recommended way to do that is like this:\n\n- ``borg repo-create repo1 --encryption=X``\n- ``borg repo-create repo2 --encryption=X --other-repo=repo1``\n- Optionally, create a snapshot to have stable and identical input data for both borg create runs.\n- client machine ---borg create---> repo1\n- client machine ---borg create---> repo2\n\nThis will create distinct (different repository ID) but related repositories.\nRelated means using the same chunker secret and the same id_key, thus producing\nthe same chunks / the same chunk IDs if the input data is the same.\n\nThe two independent borg create invocations mean there is no error propagation\nfrom repo1 to repo2 when done like that.\n\nAn alternative is to use ``borg transfer`` to copy backup archives\nfrom repo1 to repo2. This is likely a bit more efficient and the archives would be identical,\nbut it may suffer from potential error propagation.\n\nWarning: Using Borg with multiple repositories that have identical repository IDs (such as\ncreating 1:1 repository copies) is not supported and can lead to various issues,\nfor example cache coherency issues, malfunction, or data corruption.\n\n\"this is either an attack or unsafe\" warning\n--------------------------------------------\n\nAbout the warning:\n\n Cache or information obtained from the security directory is newer than the\n repository — this is either an attack or unsafe (multiple repositories with the same ID)\n\n\"unsafe\": If not following the advice from the previous section, you can easily\nrun into this by yourself by restoring an older copy of your repository.\n\n\"attack\": An attacker may have replaced your repo with an older copy, trying to\ntrigger AES counter reuse and break your repo encryption.\n\nBorg users have also reported that file system issues (e.g., hardware issues or I/O errors causing\nthe file system to become read-only) can cause this warning, see :issue:`7853`.\n\nIf you decide to ignore this and accept unsafe operation for this repository,\nyou could delete the manifest-timestamp and the local cache:\n\n::\n\n borg config id # shows the REPO_ID\n rm ~/.config/borg/security/REPO_ID/manifest-timestamp\n borg repo-delete --cache-only\n\nThis is an unsafe and unsupported way to use Borg. You have been warned.\n\nWhich file types, attributes, etc. are *not* preserved?\n-------------------------------------------------------\n\n * UNIX domain sockets (because it does not make sense - they are\n meaningless without the running process that created them and the process\n needs to recreate them in any case). So, don't panic if your backup\n misses a UDS!\n * The precise on-disk (or rather: not-on-disk) representation of the holes\n in a sparse file.\n Archive creation has no special support for sparse files, holes are\n backed up as (deduplicated and compressed) runs of zero bytes.\n Archive extraction has optional support to extract all-zero chunks as\n holes in a sparse file.\n * Some filesystem specific attributes, like btrfs NOCOW, see :ref:`platforms`.\n\nAre there other known limitations?\n----------------------------------\n\n- borg extract supports restoring only into an empty destination. After extraction,\n the destination will have exactly the contents of the extracted archive.\n If you extract into a non-empty destination, borg will (for example) not\n remove files which are in the destination, but not in the archive.\n See :issue:`4598` for a workaround and more details.\n\n.. _interrupted_backup:\n\nIf a backup stops mid-way, does the already-backed-up data stay there?\n----------------------------------------------------------------------\n\nYes, the data transferred into the repo stays there - just avoid running\n``borg compact`` before you completed the backup, because that would remove\nchunks that were already transferred to the repo, but not (yet) referenced\nby an archive.\n\nIf a backup was interrupted, you normally do not need to do anything special,\njust invoke ``borg create`` as you always do. You may use the same archive name\nas in previous attempt or a different one (e.g. if you always include the\ncurrent datetime), it does not matter.\n\nBorg always does full single-pass backups, so it will start again\nfrom the beginning - but it will be much faster, because some of the data was\nalready stored into the repo, so it does not need to get transmitted and stored\nagain.\n\n\nHow can I back up huge file(s) over an unstable connection?\n-----------------------------------------------------------\n\nYes. For more details, see :ref:`interrupted_backup`.\n\nHow can I restore huge file(s) over an unstable connection?\n-----------------------------------------------------------\n\nTry using ``borg mount`` and ``rsync`` (or a similar tool that supports\nresuming a partial file copy from what's already copied).\n\nMy machine goes to sleep causing `Broken pipe`\n----------------------------------------------\n\nWhile backing up your data over the network, your machine should not go to sleep.\nOn Linux you can use `systemd-inhibit` to avoid that. On macOS you can use `caffeinate`.\n\n``systemd-inhibit borg create ...``\n\n``caffeinate -i borg create ...``\n\nHow can I compare contents of an archive to my local filesystem?\n-----------------------------------------------------------------\n\nYou can instruct ``export-tar`` to send a tar stream to the stdout, and\nthen use ``tar`` to perform the comparison:\n\n::\n\n borg export-tar archive-name - | tar --compare -f - -C /path/to/compare/to\n\n\nCan Borg add redundancy to the backup data to deal with hardware malfunction?\n-----------------------------------------------------------------------------\n\nNo, it can't. While that at first sounds like a good idea to defend against\nsome defect HDD sectors or SSD flash blocks, dealing with this in a\nreliable way needs a lot of low-level storage layout information and\ncontrol which we do not have (and also can't get, even if we wanted).\n\nSo, if you need that, consider RAID or a filesystem that offers redundant\nstorage or just make backups to different locations / different hardware.\n\nSee also :issue:`225`.\n\nCan Borg verify data integrity of a backup archive?\n---------------------------------------------------\n\nYes, if you want to detect accidental data damage (like bit rot), use the\n``check`` operation. It will notice corruption using CRCs and hashes.\nIf you want to be able to detect malicious tampering also, use an encrypted\nrepo. It will then be able to check using CRCs and HMACs.\n\n.. _faq-integrityerror:\n\nI get an IntegrityError or similar - what now?\n----------------------------------------------\n\nA single error does not necessarily indicate bad hardware or a Borg\nbug. All hardware exhibits a bit error rate (BER). Hard drives are typically\nspecified as exhibiting fewer than one error every 12 to 120 TB\n(one bit error in 10e14 to 10e15 bits). The specification is often called\n*unrecoverable read error rate* (URE rate).\n\nApart from these very rare errors there are two main causes of errors:\n\n(i) Defective hardware: described below.\n(ii) Bugs in software (Borg, operating system, libraries):\n Ensure software is up to date.\n Check whether the issue is caused by any fixed bugs described in\n :ref:`important_notes`.\n\n.. rubric:: Finding defective hardware\n\n.. note::\n\n Hardware diagnostics are operating system dependent and do not\n apply universally. The commands shown apply for popular Unix-like\n systems. Refer to your operating system's manual.\n\nChecking hard drives\n Find the drive containing the repository and use *findmnt*, *mount* or *lsblk*\n to learn the device path (typically */dev/...*) of the drive.\n Then, smartmontools can retrieve self-diagnostics of the drive in question::\n\n # smartctl -a /dev/sdSomething\n\n The *Offline_Uncorrectable*, *Current_Pending_Sector* and *Reported_Uncorrect*\n attributes indicate data corruption. A high *UDMA_CRC_Error_Count* usually\n indicates a bad cable.\n\n I/O errors logged by the system (refer to the system journal or\n dmesg) can point to issues as well. I/O errors only affecting the\n file system easily go unnoticed, since they are not reported to\n applications (e.g. Borg), while these errors can still corrupt data.\n\n Drives can corrupt some sectors in one event, while remaining\n reliable otherwise. Conversely, drives can fail completely with no\n advance warning. If in doubt, copy all data from the drive in\n question to another drive -- just in case it fails completely.\n\n If any of these are suspicious, a self-test is recommended::\n\n # smartctl -t long /dev/sdSomething\n\n Running ``fsck`` if not done already might yield further insights.\n\nChecking memory\n Intermittent issues, such as ``borg check`` finding errors\n inconsistently between runs, are frequently caused by bad memory.\n\n Run memtest86+ (or an equivalent memory tester) to verify that\n the memory subsystem is operating correctly.\n\nChecking processors\n Processors rarely cause errors. If they do, they are usually overclocked\n or otherwise operated outside their specifications. We do not recommend to\n operate hardware outside its specifications for productive use.\n\n Tools to verify correct processor operation include Prime95 (mprime), linpack,\n and stress-ng.\n\n.. rubric:: Repairing a damaged repository\n\nWith any defective hardware found and replaced, the damage done to the repository\nneeds to be ascertained and fixed.\n\n:ref:`borg_check` provides diagnostics and ``--repair`` options for repositories with\nissues. We recommend to first run without ``--repair`` to assess the situation.\nIf the found issues and proposed repairs seem right, re-run \"check\" with ``--repair`` enabled.\n\nHow probable is it to get a hash collision problem?\n---------------------------------------------------\n\nIf you noticed, there are some issues (:issue:`170` (**warning: hell**) and :issue:`4884`)\nabout the probability of a chunk having the same hash as another chunk, making the file\ncorrupted because it grabbed the wrong chunk. This is called the `Birthday Problem\n`_.\n\nThere is a lot of probability in here so, I can give you my interpretation of\nsuch math but it's honestly better that you read it yourself and grab your own\nresolution from that.\n\nAssuming that all your chunks have a size of :math:`2^{21}` bytes (approximately 2.1 MB)\nand we have a \"perfect\" hash algorithm, we can think that the probability of collision\nwould be of :math:`p^2/2^{n+1}` then, using SHA-256 (:math:`n=256`) and for example\nwe have 1000 million chunks (:math:`p=10^9`) (1000 million chunks would be about 2100TB).\nThe probability would be around 0.0000000000000000000000000000000000000000000000000000000000043.\n\nA mass-murderer space rock happens about once every 30 million years on average.\nThis leads to a probability of such an event occurring in the next second to about :math:`10^{-15}`.\nThat's **45** orders of magnitude more probable than the SHA-256 collision. Briefly stated,\nif you find SHA-256 collisions scary then your priorities are wrong. This example was grabbed from\n`this SO answer `_, it's great honestly.\n\nStill, the real question is whether Borg tries not to make this happen?\n\nWell... previously it did not check anything until there was a feature added which saves the size\nof the chunks too, so the size of the chunks is compared to the size that you got with the\nhash and if the check says there is a mismatch it will raise an exception instead of corrupting\nthe file. This doesn't save us from everything but reduces the chances of corruption.\nThere are other ways of trying to escape this but it would affect performance so much that\nit wouldn't be worth it and it would contradict Borg's design, so if you don't want this to\nhappen, simply don't use Borg.\n\nWhy is the time elapsed in the archive stats different from wall clock time?\n----------------------------------------------------------------------------\n\nBorg needs to write the time elapsed into the archive metadata before finalizing\nthe archive and saving the files cache.\nThis means when Borg is run with e.g. the ``time`` command, the duration shown\nin the archive stats may be shorter than the full time the command runs for.\n\nHow do I configure different prune policies for different directories?\n----------------------------------------------------------------------\n\nSay you want to prune ``/var/log`` faster than the rest of\n``/``. How do we implement that? The answer is to back up to different\narchive *series* and then implement different prune policies for the\ndifferent series. For example, you could have a script that does::\n\n borg create --exclude var/log main /\n borg create logs /var/log\n\nThen you would have two different prune calls with different policies::\n\n borg prune --verbose --list -d 30 main\n borg prune --verbose --list -d 7 logs\n\nThis will keep 7 days of logs and 30 days of everything else.\n\nHow do I remove files from an existing backup?\n----------------------------------------------\n\nA file is only removed from a BorgBackup repository if all archives that contain\nthe file are deleted and the corresponding data chunks are removed from the\nrepository. There are two ways how to remove files from a repository.\n\n1. Use :ref:`borg_delete` to remove all archives that contain the files. This\nwill of course delete everything in the archive, not only some files.\n\n2. If you really want to remove only some specific files, you can run the\n:ref:`borg_recreate` command to rewrite all archives with a different\n``--exclude`` pattern. See the examples in the manpage for more information.\n\nFinally, run :ref:`borg_compact` to delete the data chunks from the repository.\n\nCan I safely change the compression level or algorithm?\n--------------------------------------------------------\n\nThe compression level and algorithm don't affect deduplication. Chunk ID hashes\nare calculated *before* compression. New compression settings\nwill only be applied to new chunks, not existing chunks. So it's safe\nto change them.\n\nUse ``borg repo-compress`` to efficiently recompress a complete repository.\n\nWhy is backing up an unmodified FAT filesystem slow on Linux?\n-------------------------------------------------------------\n\nBy default, the files cache used by BorgBackup considers the inode of files.\nWhen an inode number changes compared to the last backup, it hashes the file\nagain. The ``vfat`` kernel driver does not produce stable inode numbers by\ndefault. One way to achieve stable inode numbering is mounting the filesystem\nusing ``nfs=nostale_ro``. Doing so implies mounting the filesystem read-only.\nAnother option is to not consider inode numbers in the files cache by passing\n``--files-cache=ctime,size``.\n\nWhy are backups slow on a Linux server that is a member of a Windows domain?\n----------------------------------------------------------------------------\n\nIf a Linux server is a member of a Windows domain, username to userid resolution might be\nperformed via ``winbind`` without caching, which can slow down backups significantly.\nYou can use e.g. ``nscd`` to add caching and improve the speed.\n\nSecurity\n########\n\n.. _home_config_borg:\n\nHow important is the borg config directory?\n-------------------------------------------\n\nThe borg config directory (``~/.config/borg`` on Linux,\n``~/Library/Application Support/borg`` on macOS,\n``C:\\Users\\\\AppData\\Roaming\\borg`` on Windows -- see :ref:`env_vars`)\nhas content that you should take care of:\n\n``keys`` subdirectory\n All your borg keyfile keys are stored in this directory. Please note that borg\n repokey keys are stored inside the repository. In any case, you MUST make sure\n to have an independent backup of the borg keys, see :ref:`borg_key_export` for\n more details.\n\nMake sure that only you have access to the borg config directory.\n\n\nNote about creating multiple keyfile repositories at the same path\n------------------------------------------------------------------\n\nIf you create a new keyfile-encrypted repository at the same filesystem\npath multiple times (for example, when a previous repository at that path\nwas moved away or unmounted), Borg will not overwrite or reuse the existing\nkey file in your keys directory. Instead, it creates a new key file by\nappending a numeric suffix to the base name (e.g., .2, .3, ...).\n\nThis means you may see multiple key files like (example paths for Linux):\n\n- ~/.config/borg/keys/home_user_backup\n- ~/.config/borg/keys/home_user_backup.2\n- ~/.config/borg/keys/home_user_backup.3\n\nEach of these corresponds to a distinct repository created at the same\npath at different times. This behavior avoids accidental key reuse or\noverwrite.\n\n.. _home_data_borg:\n\nHow important is the borg data directory?\n-----------------------------------------\n\nThe borg data directory (``~/.local/share/borg`` on Linux,\n``~/Library/Application Support/borg`` on macOS,\n``C:\\Users\\\\AppData\\Local\\borg`` on Windows -- see :ref:`env_vars`)\nhas content that you should take care of:\n\n``security`` subdirectory\n Each directory here represents one Borg repository by its ID and contains the last known status.\n If a repository's status is different from this information at the beginning of BorgBackup\n operation, Borg outputs warning messages and asks for confirmation, so make sure you do not lose\n or manipulate these files. However, apart from those warnings, a loss of these files can be\n recovered.\n\nMake sure that only you have access to the Borg data directory.\n\n.. _cache_security:\n\nDo I need to take security precautions regarding the cache?\n-----------------------------------------------------------\n\nThe cache contains a lot of metadata information about the files in\nyour repositories and it is not encrypted.\n\nHowever, the assumption is that the cache is being stored on the very\nsame system which also contains the original files which are being\nbacked up. So someone with access to the cache files would also have\naccess the original files anyway.\n\nThe Internals section contains more details about :ref:`cache`. If you ever need to move the cache\nto a different location, this can be achieved by using the appropriate :ref:`env_vars`.\n\nHow can I specify the encryption passphrase programmatically?\n-------------------------------------------------------------\n\nThere are several ways to specify a passphrase without human intervention:\n\nSetting ``BORG_PASSPHRASE``\n The passphrase can be specified using the ``BORG_PASSPHRASE`` environment variable.\n This is often the simplest option, but can be insecure if the script that sets it\n is world-readable.\n\n .. _password_env:\n .. note:: Be careful how you set the environment; using the ``env``\n command, a ``system()`` call or using inline shell scripts\n (e.g. ``BORG_PASSPHRASE=hunter2 borg ...``)\n might expose the credentials in the process list directly\n and they will be readable to all users on a system. Using\n ``export`` in a shell script file should be safe, however, as\n the environment of a process is `accessible only to that\n user\n `_.\n\nUsing ``BORG_PASSCOMMAND`` with a file of proper permissions\n Another option is to create a file with a password in it in your home\n directory and use permissions to keep anyone else from reading it. For\n example, first create a key::\n\n (umask 0077; head -c 32 /dev/urandom | base64 -w 0 > ~/.borg-passphrase)\n\n Then in an automated script one can put::\n\n export BORG_PASSCOMMAND=\"cat $HOME/.borg-passphrase\"\n\n and Borg will automatically use that passphrase.\n\nUsing keyfile-based encryption with a blank passphrase\n It is possible to encrypt your repository in ``keyfile`` mode instead of the default\n ``repokey`` mode and use a blank passphrase for the key file (simply press Enter twice\n when ``borg repo-create`` asks for the password). See :ref:`encrypted_repos`\n for more details.\n\nUsing ``BORG_PASSCOMMAND`` with macOS Keychain\n macOS has a native manager for secrets (such as passphrases) which is safer\n than just using a file as it is encrypted at rest and unlocked manually\n (fortunately, the login keyring automatically unlocks when you log in). With\n the built-in ``security`` command, you can access it from the command line,\n making it useful for ``BORG_PASSCOMMAND``.\n\n First generate a passphrase and use ``security`` to save it to your login\n (default) keychain::\n\n security add-generic-password -D secret -U -a $USER -s borg-passphrase -w $(head -c 32 /dev/urandom | base64 -w 0)\n\n In your backup script retrieve it in the ``BORG_PASSCOMMAND``::\n\n export BORG_PASSCOMMAND=\"security find-generic-password -a $USER -s borg-passphrase -w\"\n\nUsing ``BORG_PASSCOMMAND`` with GNOME Keyring\n GNOME also has a keyring daemon that can be used to store a Borg passphrase.\n First ensure ``libsecret-tools``, ``gnome-keyring`` and ``libpam-gnome-keyring``\n are installed. If ``libpam-gnome-keyring`` wasn't already installed, ensure it\n runs on login::\n\n sudo sh -c \"echo session optional pam_gnome_keyring.so auto_start >> /etc/pam.d/login\"\n sudo sh -c \"echo password optional pam_gnome_keyring.so >> /etc/pam.d/passwd\"\n # you may need to relogin afterwards to activate the login keyring\n\n Then add a secret to the login keyring::\n\n head -c 32 /dev/urandom | base64 -w 0 | secret-tool store borg-repository repo-name --label=\"Borg Passphrase\"\n\n If a dialog box pops up prompting you to pick a password for a new keychain, use your\n login password. If there is a checkbox for automatically unlocking on login, check it\n to allow backups without any user intervention whatsoever.\n\n Once the secret is saved, retrieve it in a backup script using ``BORG_PASSCOMMAND``::\n\n export BORG_PASSCOMMAND=\"secret-tool lookup borg-repository repo-name\"\n\n .. note:: For this to unlock the keychain automatically it must be run\n in the ``dbus`` session of an unlocked terminal; for example, running a backup\n script as a ``cron`` job might not work unless you also ``export DISPLAY=:0``\n so ``secret-tool`` can pick up your open session. `It gets even more complicated`__\n when you are running the tool as a different user (e.g. running a backup as root\n with the password stored in the user keyring).\n\n__ https://github.com/borgbackup/borg/pull/2837#discussion_r127641330\n\nUsing ``BORG_PASSCOMMAND`` with KWallet\n KDE also has a keychain feature in the form of KWallet. The command-line tool\n ``kwalletcli`` can be used to store and retrieve secrets. Ensure ``kwalletcli``\n is installed, generate a passphrase, and store it in your \"wallet\"::\n\n head -c 32 /dev/urandom | base64 -w 0 | kwalletcli -Pe borg-passphrase -f Passwords\n\n Once the secret is saved, retrieve it in a backup script using ``BORG_PASSCOMMAND``::\n\n export BORG_PASSCOMMAND=\"kwalletcli -e borg-passphrase -f Passwords\"\n\nWhen backing up to remote encrypted repos, is encryption done locally?\n----------------------------------------------------------------------\n\nYes, file and directory metadata and data is locally encrypted, before\nleaving the local machine. We do not mean the transport layer encryption\nby that, but the data/metadata itself. Transport layer encryption (e.g.\nwhen ssh is used as a transport) applies additionally.\n\nWhen backing up to remote servers, do I have to trust the remote server?\n------------------------------------------------------------------------\n\nYes and No.\n\nNo, as far as data confidentiality is concerned - if you use encryption,\nall your files/dirs data and metadata are stored in their encrypted form\ninto the repository.\n\nYes, as an attacker with access to the remote server could delete (or\notherwise make unavailable) all your backups on that server.\n\nHow can I protect against a hacked backup client?\n-------------------------------------------------\n\nAssume you back up your backup client machine C to the backup server S and\nC gets hacked. In a simple push setup, the attacker could then use borg on\nC to delete all backups residing on S.\n\nThese are your options to protect against that:\n\n- Use a pull-mode setup using ``ssh -R``, see :ref:`pull_backup` for more information.\n- Mount C's filesystem on another machine and then create a backup of it.\n- Do not give C filesystem-level access to S.\n\nSee :ref:`hosting_repositories` for a detailed protection guide.\n\nHow can I protect against a hacked backup server?\n-------------------------------------------------\n\nJust in case you got the impression that pull-mode backups are way more safe\nthan push-mode, you also need to consider the case that your backup server S\ngets hacked. In case S has access to a lot of clients C, that might bring you\ninto even bigger trouble than a hacked backup client in the previous FAQ entry.\n\nThese are your options to protect against that:\n\n- Use the standard push-mode setup (see also previous FAQ entry).\n- Mount (the repo part of) S's filesystem on C.\n- Do not give S file-system level access to C.\n- Have your backup server at a well protected place (maybe not reachable from\n the internet), configure it safely, apply security updates, monitor it, ...\n\nHow can I protect against theft, sabotage, lightning, fire, ...?\n----------------------------------------------------------------\n\nIn general: if your only backup medium is nearby the backed-up machine and\nalways connected, you can easily get into trouble: they likely share the same\nfate if something goes really wrong.\n\nThus:\n\n- have multiple backup media\n- have media disconnected from network, power, computer\n- have media at another place\n- have a relatively recent backup on your media\n\nHow do I report a security issue with Borg?\n-------------------------------------------\n\nSend a private email to the :ref:`security contact `\nif you think you have discovered a security issue.\nPlease disclose security issues responsibly.\n\nCommon issues\n#############\n\nCommand error: ... needed to match precisely one archive, but matched N.\n------------------------------------------------------------------------\n\nThe command wanted one archive to work with. But the parameters you gave either\ndidn't match anything or they matched multiple archives.\n\nSee :ref:`archive_specification` about how to refer to an archive.\n\n/path/to/repo is not a valid repository. Check repo config.\n-----------------------------------------------------------\n\nThere can be many causes of this error. E.g. you have incorrectly specified the repository path.\n\nYou will also get this error if you try to access a repository with a key that uses the argon2 key algorithm using an old version of borg.\nWe recommend upgrading to the latest stable version and trying again. We are sorry. We should have thought about forward\ncompatibility and implemented a more helpful error message.\n\nWhy am I seeing idle borg serve processes on the repo server?\n-------------------------------------------------------------\n\nPlease see the next question.\n\nWhy does Borg disconnect or hang when backing up to a remote server?\n--------------------------------------------------------------------\n\nCommunication with the remote server (using an ssh: repo URL) happens via an SSH\nconnection. This can lead to some issues that would not occur during a local backup:\n\n- Since Borg does not send data all the time, the connection may get closed, leading\n to errors like \"connection closed by remote\".\n- On the other hand, network issues may lead to a dysfunctional connection\n that is only detected after some time by the server, leading to stale ``borg serve``\n processes and locked repositories.\n\nTo fix such problems, please apply these :ref:`SSH settings ` so that\nkeep-alive requests are sent regularly.\n\nHow can I deal with my very unstable SSH connection?\n----------------------------------------------------\n\nIf you have issues with lost connections during long-running borg commands, you\ncould try to work around:\n\n- Make partial extracts like ``borg extract PATTERN`` to do multiple\n smaller extraction runs that complete before your connection has issues.\n- Try using ``borg mount MOUNTPOINT`` and ``rsync -avH`` from\n ``MOUNTPOINT`` to your desired extraction directory. If the connection breaks\n down, just repeat that over and over again until rsync does not find anything\n to do any more. Due to the way borg mount works, this might be less efficient\n than borg extract for bigger volumes of data.\n\nCan I back up my root partition (/) with Borg?\n----------------------------------------------\n\nBacking up your entire root partition works just fine, but remember to\nexclude directories that make no sense to back up, such as /dev, /proc,\n/sys, /tmp and /run, and to use ``--one-file-system`` if you only want to\nback up the root partition (and not any mounted devices e.g.).\n\nIf it crashes with a UnicodeError, what can I do?\n-------------------------------------------------\n\nCheck if your encoding is set correctly. For most POSIX-like systems, try::\n\n export LANG=en_US.UTF-8 # or similar, important is correct charset\n\nIf that does not help:\n\n- check for typos, check if you really used ``export``.\n- check if you have set ``LC_ALL`` - if so, try not setting it.\n- check if you generated the respective locale via ``locale-gen``.\n\nI can't extract non-ascii filenames by giving them on the commandline!?\n-----------------------------------------------------------------------\n\nThis might be due to different ways to represent some characters in unicode\nor due to other non-ascii encoding issues.\n\nIf you run into that, try this:\n\n- avoid the non-ascii characters on the commandline by e.g. extracting\n the parent directory (or even everything)\n- mount the repo using FUSE and use some file manager\n\n.. _expected_performance:\n\nWhat's the expected backup performance?\n---------------------------------------\n\nCompared to simply copying files (e.g. with ``rsync``), Borg has more work to do.\nThis can make creation of the first archive slower, but saves time\nand disk space on subsequent runs. Here what Borg does when you run ``borg create``:\n\n- Borg chunks the file (using the relatively expensive buzhash algorithm)\n- It then computes the \"id\" of the chunk (hmac-sha256 (slow, except\n if your CPU has sha256 acceleration) or blake2b (fast, in software))\n- Then it checks whether this chunk is already in the repo (local hashtable lookup,\n fast). If so, the processing of the chunk is completed here. Otherwise it needs to\n process the chunk:\n- Compresses (the default lz4 is super fast)\n- Encrypts and authenticates (AES-OCB, usually fast if your CPU has AES acceleration as usual\n since about 10y, or chacha20-poly1305, fast pure-software crypto)\n- Transmits to repo. If the repo is remote, this usually involves an SSH connection\n (does its own encryption / authentication).\n- Stores the chunk into a key/value store (the key is the chunk id, the value\n is the data). While doing that, it computes XXH64 of the data (repo low-level\n checksum, used by borg check --repository).\n\nSubsequent backups are usually very fast if most files are unchanged and only\na few are new or modified. The high performance on unchanged files primarily depends\nonly on a few factors (like FS recursion + metadata reading performance and the\nfiles cache working as expected) and much less on other factors.\n\nE.g., for this setup:\n\n- server grade machine (4C/8T 2013 Xeon, 64GB RAM, 2x good 7200RPM disks)\n- local zfs filesystem (mirrored) containing the backup source data\n- repository is remote (does not matter much for unchanged files)\n- backup job runs while machine is otherwise idle\n\nThe observed performance is that Borg can process about\n**1 million unchanged files (and a few small changed ones) in 4 minutes!**\n\nIf you are seeing much less than that in similar circumstances, read the next\nfew FAQ entries below.\n\n.. _slow_backup:\n\nWhy is my backup so slow?\n--------------------------\n\nIf you feel your Borg backup is too slow somehow, here is what you can do:\n\n- Make sure Borg has enough RAM (depends on how big your repo is / how many\n files you have)\n- Use one of the blake2 modes for --encryption except if you positively know\n your CPU (and openssl) accelerates sha256 (then stay with hmac-sha256).\n- Don't use any expensive compression. The default is lz4 and super fast.\n Uncompressed is often slower than lz4.\n- Just wait. You can also interrupt it and start it again as often as you like,\n it will converge against a valid \"completed\" state. It is starting\n from the beginning each time, but it is still faster then as it does not store\n data into the repo which it already has there.\n- If you don’t need additional file attributes, you can disable them with ``--noflags``,\n ``--noacls``, ``--noxattrs``. This can lead to noticeable performance improvements\n when your backup consists of many small files.\n\nTo see what files have changed and take more time processing, you can also add\n``--list --filter=AME --stats`` to your ``borg create`` call to produce more log output,\nincluding a file list (with file status characters) and also some statistics at\nthe end of the backup.\n\nThen you do the backup and look at the log output:\n\n- stats: Do you really have little changes or are there more changes than you thought?\n In the stats you can see the overall volume of changed data, which needed to be\n added to the repo. If that is a lot, that can be the reason why it is slow.\n- ``A`` status (\"added\") in the file list:\n If you see that often, you have a lot of new files (files that Borg did not find\n in the files cache). If you think there is something wrong with that (the file was there\n already in the previous backup), please read the FAQ entries below.\n- ``M`` status (\"modified\") in the file list:\n If you see that often, Borg thinks that a lot of your files might be modified\n (Borg found them in the files cache, but the metadata read from the filesystem did\n not match the metadata stored in the files cache).\n In such a case, Borg will need to process the files' contents completely, which is\n much slower than processing unmodified files (Borg does not read their contents!).\n The metadata values used in this comparison are determined by the ``--files-cache`` option\n and could be e.g. size, ctime and inode number (see the ``borg create`` docs for more\n details and potential issues).\n You can use the ``stat`` command on files to look at fs metadata manually to debug if\n there is any unexpected change triggering the ``M`` status.\n Also, the ``--debug-topic=files_cache`` option of ``borg create`` provides a lot of debug\n output helping to analyse why the files cache does not give its expected high performance.\n\nWhen borg runs inside a virtual machine, there are some more things to look at:\n\nSome hypervisors (e.g. kvm on older proxmox) give some broadly compatible CPU type to the\nVM (usually to ease migration between VM hosts of potentially different hardware CPUs).\n\nIt is broadly compatible because they leave away modern CPU features that could be\nnot present in older or other CPUs, e.g. hardware acceleration for AES crypto, for\nsha2 hashes, for (P)CLMUL(QDQ) computations useful for crc32.\n\nSo, basically you pay for compatibility with bad performance. If you prefer better\nperformance, you should try to use the x86-64-v2-AES VCPU or the \"host\" VCPU,\nexposing misc. hw acceleration features to the VM which runs borg.\n\nOn Linux, check ``/proc/cpuinfo`` for the CPU flags inside the VM.\nFor kvm check the docs about \"Host model\" and \"Host passthrough\".\n\nSee also the next few FAQ entries for more details.\n\n.. _a_status_oddity:\n\nI am seeing 'A' (added) status for an unchanged file!?\n------------------------------------------------------\n\nThe files cache is used to determine whether Borg already\n\"knows\" / has backed up a file and if so, to skip the file from\nchunking. It intentionally *excludes* files that have a timestamp\nwhich is the same as the newest timestamp in the created archive.\n\nSo, if you see an 'A' status for unchanged file(s), they are likely the files\nwith the most recent timestamp in that archive.\n\nThis is expected: it is to avoid data loss with files that are backed up from\na snapshot and that are immediately changed after the snapshot (but within\ntimestamp granularity time, so the timestamp would not change). Without the code that\nremoves these files from the files cache, the change that happened right after\nthe snapshot would not be contained in the next backup as Borg would\nthink the file is unchanged.\n\nThis does not affect deduplication, the file will be chunked, but as the chunks\nwill often be the same and already stored in the repo (except in the above\nmentioned rare condition), it will just re-use them as usual and not store new\ndata chunks.\n\nIf you want to avoid unnecessary chunking, just create or touch a small or\nempty file in your backup source file set (so that one has the latest timestamp,\nnot your 50GB VM disk image) and, if you do snapshots, do the snapshot after\nthat.\n\nSince only the files cache is used in the display of files status,\nthose files are reported as being added when, really, chunks are\nalready used.\n\nBy default, ctime (change time) is used for the timestamps to have a rather\nsafe change detection (see also the --files-cache option).\n\nFurthermore, pathnames used as key into the files cache are **as archived**,\nso make sure these are always the same (see ``borg list``).\n\n.. _always_chunking:\n\nIt always chunks all my files, even unchanged ones!\n---------------------------------------------------\n\nBorg maintains a files cache where it remembers the timestamps, size and\ninode of files. When Borg does a new backup and starts processing a\nfile, it first looks whether the file has changed (compared to the values\nstored in the files cache). If the values are the same, the file is assumed\nunchanged and thus its contents won't get chunked (again).\n\nThe files cache is stored separately (using a different filename suffix) per\narchive series, thus using always the same name for the archive is strongly\nrecommended. The \"rebuild files cache from previous archive in repo\" feature\nalso depends on that.\nAlternatively, there is also BORG_FILES_CACHE_SUFFIX which can be used to\nmanually set a custom suffix (if you can't just use the same archive name).\n\nAnother possible reason is that files don't always have the same path -\nborg uses the paths as seen in the archive when using ``borg list``.\n\nIt is possible for some filesystems, such as ``mergerfs`` or network filesystems,\nto return inconsistent inode numbers across runs, causing borg to consider them changed.\nA workaround is to set the option ``--files-cache=ctime,size`` to exclude the inode\nnumber comparison from the files cache check so that files with different inode\nnumbers won't be treated as modified.\n\nUsing a pure-python msgpack! This will result in lower performance.\n-------------------------------------------------------------------\n\nborg uses `msgpack` to serialize/deserialize data.\n\n`msgpack` has 2 implementations:\n\n- a fast one (C code compiled into a platform specific binary), and\n- a slow pure-python one.\n\nThe slow one is used if it can't successfully import the fast one.\n\nIf you use the pyinstaller-made borg \"fat binary\" which we offer on github\nreleases, it could be that you downloaded a binary that does not match the\n(g)libc on your system.\n\nBinaries made for an older glibc than the one you have on your system usually\njust work, but the opposite is not necessarily the case and can lead to misc.\nissues - like failing to load the fast msgpack code or not working at all.\n\nSo: try a binary made for an older glibc.\n\nIf you see this without using a \"fat binary\" from us, it usually means that\nmsgpack is not built / installed correctly. It could be also that the platform\nis not fully supported (so the python code works, but there is no fast binary\ncode).\n\nIs there a way to limit bandwidth with Borg?\n--------------------------------------------\n\nTo limit upload (i.e. :ref:`borg_create`) bandwidth, use the\n``--remote-ratelimit`` option.\n\nThere is no built-in way to limit *download*\n(i.e. :ref:`borg_extract`) bandwidth, but limiting download bandwidth\ncan be accomplished with pipeviewer_:\n\nCreate a wrapper script: /usr/local/bin/pv-wrapper\n\n::\n\n #!/bin/sh\n ## -q, --quiet do not output any transfer information at all\n ## -L, --rate-limit RATE limit transfer to RATE bytes per second\n RATE=307200\n pv -q -L $RATE | \"$@\"\n\nAdd BORG_RSH environment variable to use pipeviewer wrapper script with ssh.\n\n::\n\n export BORG_RSH='/usr/local/bin/pv-wrapper ssh'\n\nNow Borg will be bandwidth limited. The nice thing about ``pv`` is that you can\nchange rate-limit on the fly:\n\n::\n\n pv -R $(pidof pv) -L 102400\n\n.. _pipeviewer: https://www.ivarch.com/programs/pv.shtml\n\n\nHow can I avoid unwanted base directories getting stored into archives?\n-----------------------------------------------------------------------\n\nPossible use cases:\n\n- Another file system is mounted and you want to back it up with original paths.\n- You have created a BTRFS snapshot in a ``/.snapshots`` directory for backup.\n\nTo achieve this, run ``borg create`` within the mountpoint/snapshot directory:\n\n::\n\n # Example: Some file system mounted in /mnt/rootfs.\n cd /mnt/rootfs\n borg create rootfs_backup .\n\nAnother way (without changing the directory) is to use the slashdot hack:\n\n::\n\n borg create rootfs_backup /mnt/rootfs/./\n\n\nI am having troubles with some network/FUSE/special filesystem, why?\n--------------------------------------------------------------------\n\nBorg is doing nothing special in the filesystem, it only uses very\ncommon and compatible operations (even the locking is just \"rename\").\n\nSo, if you are encountering issues like slowness, corruption or malfunction\nwhen using a specific filesystem, please try if you can reproduce the issues\nwith a local (non-network) and proven filesystem (like ext4 on Linux).\n\nIf you can't reproduce the issue then, you maybe have found an issue within\nthe filesystem code you used (not with Borg). For this case, it is\nrecommended that you talk to the developers / support of the network fs and\nmaybe open an issue in their issue tracker. Do not file an issue in the\nBorg issue tracker.\n\nIf you can reproduce the issue with the proven filesystem, please file an\nissue in the Borg issue tracker about that.\n\n\nWhy does running 'borg check --repair' warn about data loss?\n------------------------------------------------------------\n\nRepair usually works for recovering data in a corrupted archive. However,\nit's impossible to predict all modes of corruption. In some very rare\ninstances, such as malfunctioning storage hardware, additional repo\ncorruption may occur. If you can't afford to lose the repo, it's strongly\nrecommended that you perform repair on a copy of the repo.\n\nIn other words, the warning is there to emphasize that Borg:\n - Will perform automated routines that modify your backup repository\n - Might not actually fix the problem you are experiencing\n - Might, in very rare cases, further corrupt your repository\n\nIn the case of malfunctioning hardware, such as a drive or USB hub\ncorrupting data when read or written, it's best to diagnose and fix the\ncause of the initial corruption before attempting to repair the repo. If\nthe corruption is caused by a one time event such as a power outage,\nrunning `borg check --repair` will fix most problems.\n\n\nWhy isn't there more progress / ETA information displayed?\n----------------------------------------------------------\n\nSome borg runs take quite a bit, so it would be nice to see a progress display,\nmaybe even including a ETA (expected time of \"arrival\" [here rather \"completion\"]).\n\nFor some functionality, this can be done: if the total amount of work is more or\nless known, we can display progress. So check if there is a ``--progress`` option.\n\nBut sometimes, the total amount is unknown (e.g. for ``borg create`` we just do\na single pass over the filesystem, so we do not know the total file count or data\nvolume before reaching the end). Adding another pass just to determine that would\ntake additional time and could be incorrect, if the filesystem is changing.\n\nEven if the fs does not change and we knew count and size of all files, we still\ncould not compute the ``borg create`` ETA as we do not know the amount of changed\nchunks, how the bandwidth of source and destination or system performance might\nfluctuate.\n\nYou see, trying to display ETA would be futile. The borg developers prefer to\nrather not implement progress / ETA display than doing futile attempts.\n\nSee also: https://xkcd.com/612/\n\n\nWhy am I getting 'Operation not permitted' errors when backing up on sshfs?\n---------------------------------------------------------------------------\n\nBy default, ``sshfs`` is not entirely POSIX-compliant when renaming files due to\na technicality in the SFTP protocol. Fortunately, it also provides a workaround_\nto make it behave correctly::\n\n sshfs -o workaround=rename user@host:dir /mnt/dir\n\n.. _workaround: https://unix.stackexchange.com/a/123236\n\n\nHow do I rename a repository?\n-----------------------------\n\nThere is nothing special that needs to be done, you can simply rename the\ndirectory that corresponds to the repository. However, the next time borg\ninteracts with the repository (i.e, via ``borg list``), depending on the value\nof ``BORG_RELOCATED_REPO_ACCESS_IS_OK``, borg may warn you that the repository\nhas been moved. You will be given a prompt to confirm you are OK with this.\n\nIf ``BORG_RELOCATED_REPO_ACCESS_IS_OK`` is unset, borg will interactively ask for\neach repository whether it's OK.\n\nIt may be useful to set ``BORG_RELOCATED_REPO_ACCESS_IS_OK=yes`` to avoid the\nprompts when renaming multiple repositories or in a non-interactive context\nsuch as a script. See :doc:`deployment` for an example.\n\n\nMy backup disk is full, what can I do?\n--------------------------------------\n\nBorg cannot work if you really have zero free space on the backup disk, so the\nfirst thing you must do is deleting some files to regain free disk space. See\n:ref:`about_free_space` for further details.\n\nSome Borg commands that do not change the repository might work under disk-full\nconditions, but generally this should be avoided. If your backup disk is already\nfull when Borg starts a write command like `borg create`, it will abort\nimmediately and the repository will stay as-is.\n\n\nMiscellaneous\n#############\n\nmacOS: borg mounts not shown in Finder's side bar\n-------------------------------------------------\n\nhttps://github.com/macfuse/macfuse/wiki/Mount-options#local\n\nRead the above first and use this on your own risk::\n\n borg mount -olocal REPO MOUNTPOINT\n\n\nRequirements for the borg single-file binary, esp. (g)libc?\n-----------------------------------------------------------\n\nWe try to build the binary on old, but still supported systems - to keep the\nminimum requirement for the (g)libc low. The (g)libc can't be bundled into\nthe binary as it needs to fit your kernel and OS, but Python and all other\nrequired libraries will be bundled into the binary.\n\nIf your system fulfills the minimum (g)libc requirement (see the README that\nis released with the binary), there should be no problem. If you are slightly\nbelow the required version, maybe just try. Due to the dynamic loading (or not\nloading) of some shared libraries, it might still work depending on what\nlibraries are actually loaded and used.\n\nIn the borg git repository, there is scripts/glibc_check.py that can determine\n(based on the symbols' versions they want to link to) whether a set of given\n(Linux) binaries works with a given glibc version.\n" }, { "path": "docs/global.rst.inc", "content": ".. highlight:: bash\n.. |package_dirname| replace:: borgbackup-|version|\n.. |package_filename| replace:: |package_dirname|.tar.gz\n.. |package_url| replace:: https://pypi.org/project/borgbackup/#files\n.. |git_url| replace:: https://github.com/borgbackup/borg.git\n.. _github: https://github.com/borgbackup/borg\n.. _issue tracker: https://github.com/borgbackup/borg/issues\n.. _deduplication: https://en.wikipedia.org/wiki/Data_deduplication\n.. _AES: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard\n.. _HMAC-SHA256: https://en.wikipedia.org/wiki/HMAC\n.. _SHA256: https://en.wikipedia.org/wiki/SHA-256\n.. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2\n.. _argon2: https://en.wikipedia.org/wiki/Argon2\n.. _ACL: https://en.wikipedia.org/wiki/Access_control_list\n.. _libacl: https://savannah.nongnu.org/projects/acl/\n.. _libattr: https://savannah.nongnu.org/projects/attr/\n.. _libxxhash: https://github.com/Cyan4973/xxHash\n.. _liblz4: https://github.com/Cyan4973/lz4\n.. _libzstd: https://github.com/facebook/zstd\n.. _OpenSSL: https://www.openssl.org/\n.. _`Python 3`: https://www.python.org/\n.. _Buzhash: https://en.wikipedia.org/wiki/Buzhash\n.. _msgpack: https://msgpack.org/\n.. _`msgpack-python`: https://pypi.org/project/msgpack-python/\n.. _llfuse: https://pypi.org/project/llfuse/\n.. _mfusepy: https://pypi.org/project/mfusepy/\n.. _pyfuse3: https://pypi.org/project/pyfuse3/\n.. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace\n.. _Cython: https://cython.org/\n.. _virtualenv: https://pypi.org/project/virtualenv/\n.. _python-xxhash: https://github.com/ifduyue/python-xxhash/\n" }, { "path": "docs/index.rst", "content": ".. include:: global.rst.inc\n.. highlight:: none\n\nBorg Documentation\n==================\n\n.. include:: ../README.rst\n\n.. When you add an element here, do not forget to add it to book.rst.\n\n.. toctree::\n :maxdepth: 2\n\n installation\n quickstart\n usage\n deployment\n faq\n support\n changes\n changes_1.x\n changes_0.x\n internals\n development\n authors\n" }, { "path": "docs/installation.rst", "content": ".. include:: global.rst.inc\n.. highlight:: bash\n.. _installation:\n\nInstallation\n============\n\nThere are different ways to install Borg:\n\n- :ref:`distribution-package` - easy and fast if a package is\n available from your distribution.\n- :ref:`pyinstaller-binary` - easy and fast, we provide a ready-to-use binary file\n that comes bundled with all dependencies.\n- :ref:`source-install`, either:\n\n - :ref:`windows-binary` - builds a binary file for Windows using MSYS2.\n - :ref:`pip-installation` - installing a source package with pip needs\n more installation steps and requires all dependencies with\n development headers and a compiler.\n - :ref:`git-installation` - for developers and power users who want to\n have the latest code or use revision control (each release is\n tagged).\n\n.. _distribution-package:\n\nDistribution Package\n--------------------\n\nSome distributions might offer a ready-to-use ``borgbackup``\npackage which can be installed with the package manager.\n\n.. important:: Those packages may not be up to date with the latest\n Borg releases. Before submitting a bug\n report, check the package version and compare that to\n our latest release then review :doc:`changes` to see if\n the bug has been fixed. Report bugs to the package\n maintainer rather than directly to Borg if the\n package is out of date in the distribution.\n\n.. keep this list in alphabetical order\n\n============ ============================================= =======\nDistribution Source Command\n============ ============================================= =======\nAlpine Linux `Alpine repository`_ ``apk add borgbackup``\nArch Linux `[extra]`_ ``pacman -S borg``\nDebian `Debian packages`_ ``apt install borgbackup``\nGentoo `ebuild`_ ``emerge borgbackup``\nGNU Guix `GNU Guix`_ ``guix package --install borg``\nFedora/RHEL `Fedora official repository`_ ``dnf install borgbackup``\nFreeBSD `FreeBSD ports`_ ``cd /usr/ports/archivers/py-borgbackup && make install clean``\nmacOS `Homebrew`_ | ``brew install borgbackup`` (official formula, **no** FUSE support)\n | **or**\n | ``brew install --cask macfuse`` (`private Tap`_, FUSE support)\n | ``brew install borgbackup/tap/borgbackup-fuse``\nMageia `cauldron`_ ``urpmi borgbackup``\nNetBSD `pkgsrc`_ ``pkg_add py-borgbackup``\nNixOS `.nix file`_ ``nix-env -i borgbackup``\nOpenBSD `OpenBSD ports`_ ``pkg_add borgbackup``\nOpenIndiana `OpenIndiana hipster repository`_ ``pkg install borg``\nopenSUSE `openSUSE official repository`_ ``zypper in borgbackup``\nRaspbian `Raspbian testing`_ ``apt install borgbackup``\nUbuntu `Ubuntu packages`_, `Ubuntu PPA`_ ``apt install borgbackup``\n============ ============================================= =======\n\n.. _Alpine repository: https://pkgs.alpinelinux.org/packages?name=borgbackup\n.. _[extra]: https://www.archlinux.org/packages/?name=borg\n.. _Debian packages: https://packages.debian.org/search?keywords=borgbackup&searchon=names&exact=1&suite=all§ion=all\n.. _Fedora official repository: https://packages.fedoraproject.org/pkgs/borgbackup/borgbackup/\n.. _FreeBSD ports: https://www.freshports.org/archivers/py-borgbackup/\n.. _ebuild: https://packages.gentoo.org/packages/app-backup/borgbackup\n.. _GNU Guix: https://www.gnu.org/software/guix/package-list.html#borg\n.. _pkgsrc: https://pkgsrc.se/sysutils/py-borgbackup\n.. _cauldron: https://madb.mageia.org/package/show/application/0/release/cauldron/name/borgbackup\n.. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borgbackup/default.nix\n.. _OpenBSD ports: https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/sysutils/borgbackup/\n.. _OpenIndiana hipster repository: https://pkg.openindiana.org/hipster/en/search.shtml?token=borg&action=Search\n.. _openSUSE official repository: https://software.opensuse.org/package/borgbackup\n.. _Homebrew: https://formulae.brew.sh/formula/borgbackup\n.. _private Tap: https://github.com/borgbackup/homebrew-tap\n.. _Raspbian testing: https://archive.raspbian.org/raspbian/pool/main/b/borgbackup/\n.. _Ubuntu packages: https://launchpad.net/ubuntu/+source/borgbackup\n.. _Ubuntu PPA: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup\n\nPlease ask package maintainers to build a package or, if you can package/\nsubmit it yourself, please help us with that! See :issue:`105` on\nGitHub to follow up on packaging efforts.\n\n**Current status of package in the repositories**\n\n.. start-badges\n\n|Packaging status|\n\n.. |Packaging status| image:: https://repology.org/badge/vertical-allrepos/borgbackup.svg\n :alt: Packaging status\n :target: https://repology.org/project/borgbackup/versions\n\n.. end-badges\n\n.. _pyinstaller-binary:\n\nStandalone Binary\n-----------------\n\n.. note:: Releases are signed with an OpenPGP key, see\n :ref:`security-contact` for more instructions.\n\nBorg x86/x64 AMD/Intel compatible binaries (generated with `pyinstaller`_)\nare available on the releases_ page for the following platforms:\n\n* **Linux**: glibc >= 2.28 (ok for most supported Linux releases).\n Older glibc releases are untested and may not work.\n* **macOS**: 10.12 or newer (To avoid signing issues, download the file via\n command line **or** remove the ``quarantine`` attribute after downloading:\n ``$ xattr -dr com.apple.quarantine borg-macosx64.tgz``)\n* **FreeBSD**: 12.1 (unknown whether it works for older releases)\n\nARM binaries are built by Johann Bauer, see: https://borg.bauerj.eu/\n\nTo install such a binary, just drop it into a directory in your ``PATH``,\nmake borg readable and executable for its users and then you can run ``borg``::\n\n sudo cp borg-linux64 /usr/local/bin/borg\n sudo chown root:root /usr/local/bin/borg\n sudo chmod 755 /usr/local/bin/borg\n\nOptionally you can create a symlink to have ``borgfs`` available, which is an\nalias for ``borg mount``::\n\n ln -s /usr/local/bin/borg /usr/local/bin/borgfs\n\nNote that the binary uses /tmp to unpack Borg with all dependencies. It will\nfail if /tmp has not enough free space or is mounted with the ``noexec``\noption. You can change the temporary directory by setting the ``TEMP``\nenvironment variable before running Borg.\n\nIf a new version is released, you will have to download it manually and replace\nthe old version using the same steps as shown above.\n\n.. _pyinstaller: https://www.pyinstaller.org\n.. _releases: https://github.com/borgbackup/borg/releases\n\n.. _source-install:\n\nFrom Source\n-----------\n\n.. note::\n\n Some older Linux systems (like RHEL/CentOS 5) and Python interpreter binaries\n compiled to be able to run on such systems (like Python installed via Anaconda)\n might miss functions required by Borg.\n\n This issue will be detected early and Borg will abort with a fatal error.\n\nDependencies\n~~~~~~~~~~~~\n\nTo install Borg from a source package (including pip), you have to install the\nfollowing dependencies first. For the libraries you will also need their\ndevelopment header files (sometimes in a separate `-dev` or `-devel` package).\n\n* `Python 3`_ >= 3.10.0\n* OpenSSL_ >= 1.1.1 (LibreSSL will not work)\n* libacl_ (which depends on libattr_)\n* liblz4_ >= 1.7.0 (r129)\n* libffi (required for argon2-cffi-bindings)\n* pkg-config (cli tool) - Borg uses this to discover header and library\n locations automatically. Alternatively, you can also point to them via some\n environment variables, see setup.py.\n* Some other Python dependencies, pip will automatically install them for you.\n* Optionally, if you wish to mount an archive as a FUSE filesystem, you need\n a FUSE implementation for Python:\n\n - mfusepy_ >= 3.1.0 (for fuse 2 and fuse 3, use `pip install borgbackup[mfusepy]`), or\n - pyfuse3_ >= 3.1.1 (for fuse 3, use `pip install borgbackup[pyfuse3]`), or\n - llfuse_ >= 1.3.8 (for fuse 2, use `pip install borgbackup[llfuse]`).\n - Additionally, your OS will need to have FUSE support installed\n (e.g. a package `fuse` for fuse 2 or a package `fuse3` for fuse 3 support).\n* Optionally, if you wish to use S3/B2 Backend:\n - borgstore[s3] ~= 0.3.0 (use `pip install borgbackup[s3]`)\n* Optionally, if you wish to use SFTP Backend:\n - borgstore[sftp] ~= 0.3.0 (use `pip install borgbackup[sftp]`)\n\nIf you have troubles finding the right package names, have a look at the\ndistribution specific sections below or the Vagrantfile in the git repository,\nwhich contains installation scripts for a number of operating systems.\n\nIn the following, the steps needed to install the dependencies are listed for a\nselection of platforms. If your distribution is not covered by these\ninstructions, try to use your package manager to install the dependencies.\n\nAfter you have installed the dependencies, you can proceed with steps outlined\nunder :ref:`pip-installation`.\n\nArch Linux\n++++++++++\n\nInstall the runtime and build dependencies::\n\n pacman -S python python-pip python-virtualenv openssl acl lz4 base-devel\n pacman -S fuse2 # needed for llfuse\n pacman -S fuse3 # needed for pyfuse3\n\nNote that Arch Linux specifically doesn't support\n`partial upgrades `__,\nso in case some packages cannot be retrieved from the repo, run with ``pacman -Syu``.\n\nDebian / Ubuntu\n+++++++++++++++\n\nInstall the dependencies with development headers::\n\n sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \\\n libacl1-dev \\\n libssl-dev \\\n liblz4-dev \\\n libffi-dev \\\n build-essential pkg-config\n sudo apt-get install libfuse-dev fuse # needed for llfuse\n sudo apt-get install libfuse3-dev fuse3 # needed for pyfuse3\n\nIn case you get complaints about permission denied on ``/etc/fuse.conf``: on\nUbuntu this means your user is not in the ``fuse`` group. Add yourself to that\ngroup, log out and log in again.\n\nFedora\n++++++\n\nInstall the dependencies with development headers::\n\n sudo dnf install python3 python3-devel python3-pip python3-virtualenv \\\n libacl-devel \\\n openssl-devel \\\n lz4-devel \\\n libffi-devel \\\n pkgconf\n sudo dnf install gcc gcc-c++ redhat-rpm-config\n sudo dnf install fuse-devel fuse # needed for llfuse\n sudo dnf install fuse3-devel fuse3 # needed for pyfuse3\n\nopenSUSE Tumbleweed / Leap\n++++++++++++++++++++++++++\n\nInstall the dependencies automatically using zypper::\n\n sudo zypper source-install --build-deps-only borgbackup\n\nAlternatively, you can enumerate all build dependencies in the command line::\n\n sudo zypper install python3 python3-devel \\\n libacl-devel openssl-devel liblz4-devel \\\n libffi-devel \\\n python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \\\n python3-pytest python3-setuptools python3-setuptools_scm \\\n python3-sphinx_rtd_theme gcc gcc-c++\n sudo zypper install python3-llfuse # llfuse\n\nmacOS\n+++++\n\nWhen installing borgbackup via Homebrew_, the basic dependencies are installed automatically.\n\nFor FUSE support to mount the backup archives, you need macFUSE, which is available\nvia `github `__, or Homebrew::\n\n brew install --cask macfuse\n\nWhen installing Borg via ``pip``, be sure to install the ``llfuse`` extra,\nsince macFUSE only supports FUSE API v2. Also, since Homebrew won't link\nthe installed ``openssl`` formula, point pkg-config to the correct path::\n\n PKG_CONFIG_PATH=\"/usr/local/opt/openssl@1.1/lib/pkgconfig\" pip install borgbackup[llfuse]\n\nWhen working from a borg git repo workdir, you can install dependencies using the\nBrewfile::\n\n brew install python@3.11 # can be any supported python3 version\n brew bundle install # install requirements from borg repo's ./Brewfile\n pip3 install virtualenv\n\nBe aware that for all recent macOS releases you must authorize full disk access.\nIt is no longer sufficient to run borg backups as root. If you have not yet\ngranted full disk access, and you run Borg backup from cron, you will see\nmessages such as::\n\n /Users/you/Pictures/Photos Library.photoslibrary: scandir: [Errno 1] Operation not permitted:\n\nTo fix this problem, you should grant full disk access to cron, and to your\nTerminal application. More information `can be found here\n`__.\n\nFreeBSD\n++++++++\n\nListed below are packages you will need to install Borg, its dependencies,\nand commands to make FUSE work for using the mount command.\n\n::\n\n pkg install -y python3 pkgconf\n pkg install openssl\n pkg install liblz4\n pkg install fusefs-libs # needed for llfuse\n pkg install -y git\n python3 -m ensurepip # to install pip for Python3\n To use the mount command:\n echo 'fuse_load=\"YES\"' >> /boot/loader.conf\n echo 'vfs.usermount=1' >> /etc/sysctl.conf\n kldload fuse\n sysctl vfs.usermount=1\n\n.. _windows_deps:\n\nWindows\n+++++++\n\n.. note::\n Running under Windows is experimental.\n\n.. warning::\n This script needs to be run in the UCRT64 environment in MSYS2.\n\nInstall the dependencies with the provided script::\n\n ./scripts/msys2-install-deps\n\n.. _msys2_path_translation:\n\nMSYS2 Path Translation\n++++++++++++++++++++++\n\nWhen running Borg within an MSYS2 environment, the shell\nautomatically translates POSIX-style paths (like ``/tmp`` or ``/C/Users``) to\nWindows paths (like ``C:\\msys64\\tmp`` or ``C:\\Users``) before they reach the\nBorg process.\n\nThis behavior can result in absolute Windows paths being stored in your backups,\nwhich may not be what you intended if you use POSIX paths for portability.\n\nTo disable this automatic translation for Borg, you can use environment variables\nto exclude everything from conversion. Similarly, MSYS2 also translates\nenvironment variables that look like paths. To disable this generally for Borg,\nset both variables::\n\n export MSYS2_ARG_CONV_EXCL=\"*\"\n export MSYS2_ENV_CONV_EXCL=\"*\"\n\nFor more details, see the `MSYS2 documentation on filesystem paths `__.\n\nWindows 10's Linux Subsystem\n++++++++++++++++++++++++++++\n\n.. note::\n Running under Windows 10's Linux Subsystem is experimental and has not been tested much yet.\n\nJust follow the Ubuntu Linux installation steps. You can omit the FUSE stuff, it won't work anyway.\n\n\nCygwin\n++++++\n\n.. note::\n Running under Cygwin is experimental and has not been tested much yet.\n\nUse the Cygwin installer to install the dependencies::\n\n python39 python39-devel\n python39-setuptools python39-pip python39-wheel python39-virtualenv\n libssl-devel liblz4-devel\n binutils gcc-g++ git make openssh\n\nMake sure to use a virtual environment to avoid confusions with any Python installed on Windows.\n\n.. _windows-binary:\n\nBuilding a binary on Windows\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. note::\n This is experimental.\n\n.. warning::\n This needs to be run in the UCRT64 environment in MSYS2.\n\nEnsure to install the dependencies as described within :ref:`Dependencies: Windows `.\n\n::\n\n # Needed for setuptools < 70.2.0 to work - https://www.msys2.org/docs/python/#known-issues\n # export SETUPTOOLS_USE_DISTUTILS=stdlib\n pip install -e .\n pyinstaller -y scripts/borg.exe.spec\n\nA standalone executable will be created in ``dist/borg.exe``.\n\n.. _pip-installation:\n\nUsing pip\n~~~~~~~~~\n\nVirtualenv_ can be used to build and install Borg without affecting\nthe system Python or requiring root access. Using a virtual environment is\noptional, but recommended except for the most simple use cases.\n\nEnsure to install the dependencies as described within :ref:`source-install`.\n\n.. note::\n If you install into a virtual environment, you need to **activate** it\n first (``source borg-env/bin/activate``), before running ``borg``.\n Alternatively, symlink ``borg-env/bin/borg`` into some directory that is in\n your ``PATH`` so you can run ``borg``.\n\nThis will use ``pip`` to install the latest release from PyPi::\n\n virtualenv --python=python3 borg-env\n source borg-env/bin/activate\n\n # might be required if your tools are outdated\n pip install -U pip setuptools wheel\n\n # install Borg + Python dependencies into virtualenv\n pip install borgbackup\n # or alternatively (if you want FUSE support):\n pip install borgbackup[llfuse] # to use llfuse\n pip install borgbackup[pyfuse3] # to use pyfuse3\n\nTo upgrade Borg to a new version later, run the following after\nactivating your virtual environment::\n\n pip install -U borgbackup # or ... borgbackup[llfuse/pyfuse3]\n\nWhen doing manual pip installation, man pages are not automatically\ninstalled. You can run these commands to install the man pages\nlocally::\n\n # get borg from github\n git clone https://github.com/borgbackup/borg.git borg\n\n # Install the files with proper permissions\n install -D -m 0644 borg/docs/man/borg*.1* $HOME/.local/share/man/man1/borg.1\n\n # Update the man page cache\n mandb\n\n.. _git-installation:\n\nUsing git\n~~~~~~~~~\n\nThis uses latest, unreleased development code from git.\nWhile we try not to break master, there are no guarantees on anything.\n\nEnsure to install the dependencies as described within :ref:`source-install`.\n\nVersion metadata is obtained dynamically at install time using ``setuptools-scm``.\nPlease ensure that your git repo either has correct tags, or provide the version\nmanually using the ``SETUPTOOLS_SCM_PRETEND_VERSION`` environment variable.\n\n::\n\n # get borg from github\n git clone https://github.com/borgbackup/borg.git\n\n # create a virtual environment\n virtualenv --python=$(which python3) borg-env\n source borg-env/bin/activate # always before using!\n\n # install borg dependencies into virtualenv\n cd borg\n pip install -r requirements.d/development.lock.txt\n pip install -r requirements.d/docs.txt # optional, to build the docs\n\n # set a borg version if setuptools-scm fails to do so automatically\n export SETUPTOOLS_SCM_PRETEND_VERSION=\n\n # install borg into virtualenv\n pip install -e . # in-place editable mode\n or\n pip install -e .[pyfuse3] # in-place editable mode, use pyfuse3\n or\n pip install -e .[llfuse] # in-place editable mode, use llfuse\n\n # optional: run all the tests, on all installed Python versions\n # requires fakeroot, available through your package manager\n fakeroot -u tox --skip-missing-interpreters\n\nBy default the system installation of python will be used.\nIf you need to use a different version of Python you can install this using ``pyenv``:\n\n::\n\n ...\n # create a virtual environment\n pyenv install 3.10.0 # minimum, preferably use something more recent!\n pyenv global 3.10.0\n pyenv local 3.10.0\n virtualenv --python=${pyenv which python} borg-env\n source borg-env/bin/activate # always before using!\n ...\n\n.. note:: As a developer or power user, you should always use a virtual environment.\n" }, { "path": "docs/internals/data-structures.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n\n.. _data-structures:\n\nData structures and file formats\n================================\n\nThis page documents the internal data structures and storage\nmechanisms of Borg. It is partly based on mailing list\ndiscussions and also on static code analysis.\n\n.. todo:: Clarify terms, perhaps create a glossary.\n ID (client?) vs. key (repository?),\n chunks (blob of data in repo?) vs. object (blob of data in repo, referred to from another object?),\n\n.. _repository:\n\nRepository\n----------\n\nBorg stores its data in a `Repository`, which is a key-value store and has\nthe following structure:\n\nconfig/\n readme\n simple text object telling that this is a Borg repository\n id\n the unique repository ID encoded as hexadecimal number text\n version\n the repository version encoded as decimal number text\n manifest\n some data about the repository, binary\n last-key-checked\n repository check progress (partial checks, full checks' checkpointing),\n path of last object checked as text\n space-reserve.N\n purely random binary data to reserve space, e.g. for disk-full emergencies\n\nThere is a list of pointers to archive objects in this directory:\n\narchives/\n 0000... .. ffff...\n\nThe actual data is stored into a nested directory structure, using the full\nobject ID as name. Each (encrypted and compressed) object is stored separately.\n\ndata/\n 00/ .. ff/\n 00/ .. ff/\n 0000... .. ffff...\n\nkeys/\n repokey\n When using encryption in repokey mode, the encrypted, passphrase protected\n key is stored here as a base64 encoded text.\n\nlocks/\n used by the locking system to manage shared and exclusive locks.\n\n\nKeys\n~~~~\n\nRepository object IDs (which are used as key into the key-value store) are\nbyte strings of fixed length (256-bit, 32 bytes), computed like this::\n\n key = id = id_hash(plaintext_data) # plain = not encrypted, not compressed, not obfuscated\n\nThe id_hash function depends on the :ref:`encryption mode `.\n\nAs the id / key is used for deduplication, id_hash must be a cryptographically\nstrong hash or MAC.\n\nRepository objects\n~~~~~~~~~~~~~~~~~~\n\nEach repository object is stored separately, under its ID into data/xx/yy/xxyy...\n\nA repo object has a structure like this:\n\n* 32-bit meta size\n* 32-bit data size\n* 64-bit xxh64(meta)\n* 64-bit xxh64(data)\n* meta\n* data\n\nThe size and xxh64 hashes can be used for server-side corruption checks without\nneeding to decrypt anything (which would require the borg key).\n\nThe overall size of repository objects varies from very small (a small source\nfile will be stored as a single repository object) to medium (big source files will\nbe cut into medium-sized chunks of some MB).\n\nMetadata and data are separately encrypted and authenticated (depending on\nthe user's choices).\n\nSee :ref:`data-encryption` for a graphic outlining the anatomy of the\nencryption.\n\nRepo object metadata\n~~~~~~~~~~~~~~~~~~~~\n\nMetadata is a MessagePack-encoded (and encrypted/authenticated) dict with:\n\n- ctype (compression type 0..255)\n- clevel (compression level 0..255)\n- csize (overall compressed (and maybe obfuscated) data size)\n- psize (only when obfuscated: payload size without the obfuscation trailer)\n- size (uncompressed size of the data)\n\nHaving this separately encrypted metadata makes it more efficient to query\nthe metadata without having to read, transfer and decrypt the (usually much\nbigger) data part.\n\nThe compression `ctype` and `clevel` is explained in :ref:`data-compression`.\n\n\nCompaction\n~~~~~~~~~~\n\n``borg compact`` is used to free repository space. It will:\n\n- list all object IDs present in the repository\n- read all archives and determine which object IDs are in use\n- remove all unused objects from the repository\n- inform / warn about anything remarkable it found:\n\n - warn about IDs used, but not present (data loss!)\n - inform about IDs that reappeared that were previously lost\n- compute statistics about:\n\n - compression and deduplication factors\n - repository space usage and space freed\n\n\nThe object graph\n----------------\n\nOn top of the simple key-value store offered by the Repository_,\nBorg builds a much more sophisticated data structure that is essentially\na completely encrypted object graph. Objects, such as archives_, are referenced\nby their chunk ID, which is cryptographically derived from their contents.\nMore on how this helps security in :ref:`security_structural_auth`.\n\n.. figure:: object-graph.png\n :figwidth: 100%\n :width: 100%\n\n.. _manifest:\n\nThe manifest\n~~~~~~~~~~~~\n\nCompared to borg 1.x:\n\n- the manifest moved from object ID 0 to config/manifest\n- the archives list has been moved from the manifest to archives/*\n\nThe manifest is rewritten each time an archive is created, deleted,\nor modified. It looks like this:\n\n.. code-block:: python\n\n {\n 'version': 1,\n 'timestamp': '2017-05-05T12:42:23.042864',\n 'item_keys': ['acl_access', 'acl_default', ...],\n 'config': {},\n 'archives': {\n '2017-05-05-system-backup': {\n 'id': b'<32 byte binary object ID>',\n 'time': '2017-05-05T12:42:22.942864',\n },\n },\n }\n\nThe *version* field can be either 1 or 2. The versions differ in the\nway feature flags are handled, described below.\n\nThe *timestamp* field is used to avoid logical replay attacks where\nthe server just resets the repository to a previous state.\n\n*item_keys* is a list containing all Item_ keys that may be encountered in\nthe repository. It is used by *borg check*, which verifies that all keys\nin all items are a subset of these keys. Thus, an older version of *borg check*\nsupporting this mechanism can correctly detect keys introduced in later versions.\n\n*config* is a general-purpose location for additional metadata. All versions\nof Borg preserve its contents.\n\nFeature flags\n+++++++++++++\n\nFeature flags are used to add features to data structures without causing\ncorruption if older versions are used to access or modify them. The main issues\nto consider for a feature flag oriented design are flag granularity,\nflag storage, and cache_ invalidation.\n\nFeature flags are divided in approximately three categories, detailed below.\nDue to the nature of ID-based deduplication, write (i.e. creating archives) and\nread access are not symmetric; it is possible to create archives referencing\nchunks that are not readable with the current feature set. The third\ncategory are operations that require accurate reference counts, for example\narchive deletion and check.\n\nAs the manifest is always updated and always read, it is the ideal place to store\nfeature flags, comparable to the super-block of a file system. The only problem\nis to recover from a lost manifest, i.e. how is it possible to detect which feature\nflags are enabled, if there is no manifest to tell. This issue is left open at this time,\nbut is not expected to be a major hurdle; it doesn't have to be handled efficiently, it just\nneeds to be handled.\n\nLastly, cache_ invalidation is handled by noting which feature\nflags were and which were not understood while manipulating a cache.\nThis allows borg to detect whether the cache needs to be invalidated,\ni.e. rebuilt from scratch. See `Cache feature flags`_ below.\n\nThe *config* key stores the feature flags enabled on a repository:\n\n.. code-block:: python\n\n config = {\n 'feature_flags': {\n 'read': {\n 'mandatory': ['some_feature'],\n },\n 'check': {\n 'mandatory': ['other_feature'],\n }\n 'write': ...,\n 'delete': ...\n },\n }\n\nThe top-level distinction for feature flags is the operation the client intends\nto perform,\n\n| the *read* operation includes extraction and listing of archives,\n| the *write* operation includes creating new archives,\n| the *delete* (archives) operation,\n| the *check* operation requires full understanding of everything in the repository.\n|\n\nThese are weakly set-ordered; *check* will include everything required for *delete*,\n*delete* will likely include *write* and *read*. However, *read* may require more\nfeatures than *write* (due to ID-based deduplication, *write* does not necessarily\nrequire reading/understanding repository contents).\n\nEach operation can contain several sets of feature flags. Only one set,\nthe *mandatory* set is currently defined.\n\nUpon reading the manifest, the Borg client has already determined which operation\nshould be performed. If feature flags are found in the manifest, the set\nof feature flags supported by the client is compared to the mandatory set\nfound in the manifest. If any unsupported flags are found (i.e. the mandatory set is\nnot a subset of the features supported by the Borg client used), the operation\nis aborted with a *MandatoryFeatureUnsupported* error:\n\n Unsupported repository feature(s) {'some_feature'}. A newer version of borg is required to access this repository.\n\nOlder Borg releases do not have this concept and do not perform feature flags checks.\nThese can be locked out with manifest version 2. Thus, the only difference between\nmanifest versions 1 and 2 is that the latter is only accepted by Borg releases\nimplementing feature flags.\n\nTherefore, as soon as any mandatory feature flag is enabled in a repository,\nthe manifest version must be switched to version 2 in order to lock out all\nBorg releases unaware of feature flags.\n\n.. _Cache feature flags:\n.. rubric:: Cache feature flags\n\n`The cache`_ does not have its separate set of feature flags. Instead, Borg stores\nwhich flags were used to create or modify a cache.\n\nAll mandatory manifest features from all operations are gathered in one set.\nThen, two sets of features are computed;\n\n- those features that are supported by the client and mandated by the manifest\n are added to the *mandatory_features* set,\n- the *ignored_features* set comprised of those features mandated by the manifest,\n but not supported by the client.\n\nBecause the client previously checked compliance with the mandatory set of features\nrequired for the particular operation it is executing, the *mandatory_features* set\nwill contain all necessary features required for using the cache safely.\n\nConversely, the *ignored_features* set contains only those features which were not\nrelevant to operating the cache. Otherwise, the client would not pass the feature\nset test against the manifest.\n\nWhen opening a cache and the *mandatory_features* set is not a subset of the features\nsupported by the client, the cache is wiped out and rebuilt,\nsince a client not supporting a mandatory feature that the cache was built with\nwould be unable to update it correctly.\nThe assumption behind this behaviour is that any of the unsupported features could have\nbeen reflected in the cache and there is no way for the client to discern whether\nthat is the case.\nMeanwhile, it may not be practical for every feature to have clients using it track\nwhether the feature had an impact on the cache.\nTherefore, the cache is wiped.\n\nWhen opening a cache and the intersection of *ignored_features* and the features\nsupported by the client contains any elements, i.e. the client possesses features\nthat the previous client did not have and those new features are enabled in the repository,\nthe cache is wiped out and rebuilt.\n\nWhile the former condition likely requires no tweaks, the latter condition is formulated\nin an especially conservative way to play it safe. It seems likely that specific features\nmight be exempted from the latter condition.\n\n.. rubric:: Defined feature flags\n\nCurrently no feature flags are defined.\n\nFrom currently planned features, some examples follow,\nthese may/may not be implemented and purely serve as examples.\n\n- A mandatory *read* feature could be using a different encryption scheme (e.g. session keys).\n This may not be mandatory for the *write* operation - reading data is not strictly required for\n creating an archive.\n- Any additions to the way chunks are referenced (e.g. to support larger archives) would\n become a mandatory *delete* and *check* feature; *delete* implies knowing correct\n reference counts, so all object references need to be understood. *check* must\n discover the entire object graph as well, otherwise the \"orphan chunks check\"\n could delete data still in use.\n\n.. _archive:\n\nArchives\n~~~~~~~~\n\nEach archive is an object referenced by an entry below archives/.\nThe archive object itself does not store any of the data contained in the\narchive it describes.\n\nInstead, it contains a list of chunks which form a msgpacked stream of items_.\nThe archive object itself further contains some metadata:\n\n* *version*\n* *name*, which might differ from the name set in the archives/* object.\n When :ref:`borg_check` rebuilds the manifest (e.g. if it was corrupted) and finds\n more than one archive object with the same name, it adds a counter to the name\n in archives/*, but leaves the *name* field of the archives as they were.\n* *item_ptrs*, a list of \"pointer chunk\" IDs.\n Each \"pointer chunk\" contains a list of chunk IDs of item metadata.\n* *command_line*, the command line which was used to create the archive\n* *hostname*\n* *username*\n* *time* and *time_end* are the start and end timestamps, respectively\n* *comment*, a user-specified archive comment\n* *chunker_params* are the :ref:`chunker-params ` used for creating the archive.\n This is used by :ref:`borg_recreate` to determine whether a given archive needs rechunking.\n* Some other pieces of information related to recreate.\n\n.. _item:\n\nItems\n~~~~~\n\nEach item represents a file, directory or other file system item and is stored as a\ndictionary created by the ``Item`` class that contains:\n\n* path\n* list of data chunks (size: count * ~40B)\n* user\n* group\n* uid\n* gid\n* mode (item type + permissions)\n* source (for symlinks)\n* hlid (for hardlinks)\n* rdev (for device files)\n* mtime, atime, ctime, birthtime in nanoseconds\n* xattrs\n* acl (various OS-dependent fields)\n* flags\n\nAll items are serialized using msgpack and the resulting byte stream\nis fed into the same chunker algorithm as used for regular file data\nand turned into deduplicated chunks. The reference to these chunks is then added\nto the archive metadata. To achieve a finer granularity on this metadata\nstream, we use different chunker params for this chunker, which result in\nsmaller chunks.\n\nA chunk is stored as an object as well, of course.\n\n.. _chunks:\n.. _chunker_details:\n\nChunks\n~~~~~~\n\nBorg has these chunkers:\n\n- \"fixed\": a simple, low cpu overhead, fixed blocksize chunker, optionally\n supporting a header block of different size.\n- \"buzhash\": variable, content-defined blocksize, uses a rolling hash\n computed by the Buzhash_ algorithm.\n- \"buzhash64\": similar to \"buzhash\", but improved 64bit implementation\n\nFor some more general usage hints see also ``--chunker-params``.\n\n\"fixed\" chunker\n+++++++++++++++\n\nThe fixed chunker triggers (chunks) at even-spaced offsets, e.g. every 4MiB,\nproducing chunks of same block size (the last chunk is not required to be\nfull-size).\n\nOptionally, it supports processing a differently sized \"header\" first, before\nit starts to cut chunks of the desired block size.\nThe default is not to have a differently sized header.\n\n``borg create --chunker-params fixed,BLOCK_SIZE[,HEADER_SIZE]``\n\n- BLOCK_SIZE: no default value, multiple of the system page size (usually 4096\n bytes) recommended. E.g.: 4194304 would cut 4MiB sized chunks.\n- HEADER_SIZE: optional, defaults to 0 (no header).\n\nThe fixed chunker also supports processing sparse files (reading only the ranges\nwith data and seeking over the empty hole ranges).\n\n``borg create --sparse --chunker-params fixed,BLOCK_SIZE[,HEADER_SIZE]``\n\n\"buzhash\" chunker\n+++++++++++++++++\n\nThe buzhash chunker triggers (chunks) when the last HASH_MASK_BITS bits of the\nhash are zero, producing chunks with a target size of 2^HASH_MASK_BITS bytes.\n\nBuzhash is **only** used for cutting the chunks at places defined by the\ncontent, the buzhash value is **not** used as the deduplication criteria (we\nuse a cryptographically strong hash/MAC over the chunk contents for this, the\nid_hash).\n\nThe idea of content-defined chunking is assigning every byte where a\ncut *could* be placed a hash. The hash is based on some number of bytes\n(the window size) before the byte in question. Chunks are cut\nwhere the hash satisfies some condition\n(usually \"n numbers of trailing/leading zeroes\"). This causes chunks to be cut\nin the same location relative to the file's contents, even if bytes are inserted\nor removed before/after a cut, as long as the bytes within the window stay the same.\nThis results in a high chance that a single cluster of changes to a file will only\nresult in 1-2 new chunks, aiding deduplication.\n\nUsing normal hash functions this would be extremely slow,\nrequiring hashing approximately ``window size * file size`` bytes.\nA rolling hash is used instead, which allows to add a new input byte and\ncompute a new hash as well as *remove* a previously added input byte\nfrom the computed hash. This makes the cost of computing a hash for each\ninput byte largely independent of the window size.\n\nBorg defines minimum and maximum chunk sizes (CHUNK_MIN_EXP and CHUNK_MAX_EXP, respectively)\nwhich narrows down where cuts may be made, greatly reducing the amount of data\nthat is actually hashed for content-defined chunking.\n\n``borg create --chunker-params buzhash,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE``\ncan be used to tune the chunker parameters, the default is:\n\n- CHUNK_MIN_EXP = 19 (minimum chunk size = 2^19 B = 512 kiB)\n- CHUNK_MAX_EXP = 23 (maximum chunk size = 2^23 B = 8 MiB)\n- HASH_MASK_BITS = 21 (target chunk size ~= 2^21 B = 2 MiB)\n- HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`) (must be an odd number)\n\nThe buzhash table is altered by XORing it with a seed randomly generated once\nfor the repository, and stored encrypted in the keyfile. This is to prevent\nchunk size based fingerprinting attacks on your encrypted repo contents (to\nguess what files you have based on a specific set of chunk sizes).\n\n\"buzhash64\" chunker\n+++++++++++++++++++\n\nSimilar to \"buzhash\", but using 64bit wide hash values.\n\nThe buzhash table is cryptographically derived from secret key material.\n\nThese changes should improve resistance against attacks and also solve\nsome of the issues of the original (32bit / XORed table) implementation.\n\n.. _cache:\n\nThe cache\n---------\n\nThe **files cache** is stored in ``cache/files.`` and is used at backup\ntime to quickly determine whether a given file is unchanged and we have all its\nchunks.\n\nIn memory, the files cache is a key -> value mapping (a Python *dict*) and contains:\n\n* key: id_hash of the encoded path (same path as seen in archive)\n* value:\n\n - age (0 [newest], ..., BORG_FILES_CACHE_TTL - 1)\n - file inode number\n - file size\n - file ctime_ns\n - file mtime_ns\n - list of chunk (id, size) tuples representing the file's contents\n\nTo determine whether a file has not changed, cached values are looked up via\nthe key in the mapping and compared to the current file attribute values.\n\nIf the file's size, timestamp and inode number is still the same, it is\nconsidered not to have changed. In that case, we check that all file content\nchunks are (still) present in the repository (we check that via the chunks\ncache).\n\nIf everything is matching and all chunks are present, the file is not read /\nchunked / hashed again (but still a file metadata item is written to the\narchive, made from fresh file metadata read from the filesystem). This is\nwhat makes borg so fast when processing unchanged files.\n\nIf there is a mismatch or a chunk is missing, the file is read / chunked /\nhashed. Chunks already present in repo won't be transferred to repo again.\n\nThe inode number is stored and compared to make sure we distinguish between\ndifferent files, as a single path may not be unique across different\narchives in different setups.\n\nNot all filesystems have stable inode numbers. If that is the case, borg can\nbe told to ignore the inode number in the check via --files-cache.\n\nThe age value is used for cache management. If a file is \"seen\" in a backup\nrun, its age is reset to 0, otherwise its age is incremented by one.\nIf a file was not seen in BORG_FILES_CACHE_TTL backups, its cache entry is\nremoved.\n\nThe files cache is a python dictionary, storing python objects, which\ngenerates a lot of overhead.\n\nBorg can also work without using the files cache (saves memory if you have a\nlot of files or not much RAM free), then all files are assumed to have changed.\nThis is usually much slower than with files cache.\n\nThe on-disk format of the files cache is a stream of msgpacked tuples (key, value).\nLoading the files cache involves reading the file, one msgpack object at a time,\nunpacking it, and msgpacking the value (in an effort to save memory).\n\nThe **chunks cache** is not persisted to disk, but dynamically built in memory\nby querying the existing object IDs from the repository.\nIt is used to determine whether we already have a specific chunk.\n\nThe chunks cache is a key -> value mapping and contains:\n\n* key:\n\n - chunk id_hash\n* value:\n\n - reference count (always MAX_VALUE as we do not refcount anymore)\n - size (0 for prev. existing objects, we can't query their plaintext size)\n\nThe chunks cache is a HashIndex_.\n\n.. _cache-memory-usage:\n\nIndexes / Caches memory usage\n-----------------------------\n\nHere is the estimated memory usage of Borg - it's complicated::\n\n chunk_size ~= 2 ^ HASH_MASK_BITS (for buzhash chunker, BLOCK_SIZE for fixed chunker)\n chunk_count ~= total_file_size / chunk_size\n\n chunks_cache_usage = chunk_count * 40\n\n files_cache_usage = total_file_count * 240 + chunk_count * 165\n\n mem_usage ~= chunks_cache_usage + files_cache_usage\n = chunk_count * 205 + total_file_count * 240\n\nDue to the hashtables, the best/usual/worst cases for memory allocation can\nbe estimated like that::\n\n mem_allocation = mem_usage / load_factor # l_f = 0.25 .. 0.75\n\n mem_allocation_peak = mem_allocation * (1 + growth_factor) # g_f = 1.1 .. 2\n\nAll units are Bytes.\n\nIt is assuming every chunk is referenced exactly once (if you have a lot of\nduplicate chunks, you will have fewer chunks than estimated above).\n\nIt is also assuming that typical chunk size is 2^HASH_MASK_BITS (if you have\na lot of files smaller than this statistical medium chunk size, you will have\nmore chunks than estimated above, because 1 file is at least 1 chunk).\n\nThe chunks cache and files cache are all implemented as hash tables.\nA hash table must have a significant amount of unused entries to be fast -\nthe so-called load factor gives the used/unused elements ratio.\n\nWhen a hash table gets full (load factor getting too high), it needs to be\ngrown (allocate new, bigger hash table, copy all elements over to it, free old\nhash table) - this will lead to short-time peaks in memory usage each time this\nhappens. Usually does not happen for all hashtables at the same time, though.\nFor small hash tables, we start with a growth factor of 2, which comes down to\n~1.1x for big hash tables.\n\nE.g. backing up a total count of 1 Mi (IEC binary prefix i.e. 2^20) files with a total size of 1TiB.\n\na) with ``create --chunker-params buzhash,10,23,16,4095`` (custom):\n\n mem_usage = 2.8GiB\n\nb) with ``create --chunker-params buzhash,19,23,21,4095`` (default):\n\n mem_usage = 0.31GiB\n\n.. note:: There is also the ``--files-cache=disabled`` option to disable the files cache.\n You'll save some memory, but it will need to read / chunk all the files as\n it can not skip unmodified files then.\n\nHashIndex\n---------\n\nThe chunks cache is implemented as a hash table, with\nonly one slot per bucket, spreading hash collisions to the following\nbuckets. As a consequence the hash is just a start position for a linear\nsearch. If a key is looked up that is not in the table, then the hash table\nis searched from the start position (the hash) until the first empty\nbucket is reached.\n\nThis particular mode of operation is open addressing with linear probing.\n\nWhen the hash table is filled to 75%, its size is grown. When it's\nemptied to 25%, its size is shrunken. Operations on it have a variable\ncomplexity between constant and linear with low factor, and memory overhead\nvaries between 33% and 300%.\n\nIf an element is deleted, and the slot behind the deleted element is not empty,\nthen the element will leave a tombstone, a bucket marked as deleted. Tombstones\nare only removed by insertions using the tombstone's bucket, or by resizing\nthe table. They present the same load to the hash table as a real entry,\nbut do not count towards the regular load factor.\n\nThus, if the number of empty slots becomes too low (recall that linear probing\nfor an element not in the index stops at the first empty slot), the hash table\nis rebuilt. The maximum *effective* load factor, i.e. including tombstones, is 93%.\n\nData in a HashIndex is always stored in little-endian format, which increases\nefficiency for almost everyone, since basically no one uses big-endian processors\nany more.\n\nHashIndex does not use a hashing function, because all keys (save manifest) are\noutputs of a cryptographic hash or MAC and thus already have excellent distribution.\nThus, HashIndex simply uses the first 32 bits of the key as its \"hash\".\n\nThe format is easy to read and write, because the buckets array has the same layout\nin memory and on disk. Only the header formats differ. The on-disk header is\n``struct HashHeader``:\n\n- First, the HashIndex magic, the eight byte ASCII string \"BORG_IDX\".\n- Second, the signed 32-bit number of entries (i.e. buckets which are not deleted and not empty).\n- Third, the signed 32-bit number of buckets, i.e. the length of the buckets array\n contained in the file, and the modulus for index calculation.\n- Fourth, the signed 8-bit length of keys.\n- Fifth, the signed 8-bit length of values. This has to be at least four bytes.\n\nAll fields are packed.\n\nThe HashIndex is *not* a general purpose data structure.\nThe value size must be at least 4 bytes, and these first bytes are used for in-band\nsignalling in the data structure itself.\n\nThe constant MAX_VALUE (defined as 2**32-1025 = 4294966271) defines the valid range for\nthese 4 bytes when interpreted as an uint32_t from 0 to MAX_VALUE (inclusive).\nThe following reserved values beyond MAX_VALUE are currently in use (byte order is LE):\n\n- 0xffffffff marks empty buckets in the hash table\n- 0xfffffffe marks deleted buckets in the hash table\n\nHashIndex is implemented in C and wrapped with Cython in a class-based interface.\nThe Cython wrapper checks every passed value against these reserved values and\nraises an AssertionError if they are used.\n\n.. _data-encryption:\n\nEncryption\n----------\n\n.. seealso:: The :ref:`borgcrypto` section for an in-depth review.\n\nAEAD modes\n~~~~~~~~~~\n\nFor new repositories, borg only uses modern AEAD ciphers: AES-OCB or CHACHA20-POLY1305.\n\nFor each borg invocation, a new sessionkey is derived from the borg key material\nand the 48bit IV starts from 0 again (both ciphers internally add a 32bit counter\nto our IV, so we'll just count up by 1 per chunk).\n\nThe encryption layout is best seen at the bottom of this diagram:\n\n.. figure:: encryption-aead.png\n :figwidth: 100%\n :width: 100%\n\nNo special IV/counter management is needed here due to the use of session keys.\n\nA 48 bit IV is way more than needed: If you only backed up 4kiB chunks (2^12B),\nthe IV would \"limit\" the data encrypted in one session to 2^(12+48)B == 2.3 exabytes,\nmeaning you would run against other limitations (RAM, storage, time) way before that.\nIn practice, chunks are usually bigger, for big files even much bigger, giving an\neven higher limit.\n\nLegacy modes\n~~~~~~~~~~~~\n\nOld repositories (which used AES-CTR mode) are supported read-only to be able to\n``borg transfer`` their archives to new repositories (which use AEAD modes).\n\nAES-CTR mode is not supported for new repositories and the related code will be\nremoved in a future release.\n\nBoth modes\n~~~~~~~~~~\n\nEncryption keys (and other secrets) are kept either in a key file on the client\n('keyfile' mode) or in the repository under keys/repokey ('repokey' mode).\nIn both cases, the secrets are generated from random and then encrypted by a\nkey derived from your passphrase (this happens on the client before the key\nis stored into the keyfile or as repokey).\n\nThe passphrase is passed through the ``BORG_PASSPHRASE`` environment variable\nor prompted for interactive usage.\n\n.. _key_files:\n\nKey files\n---------\n\n.. seealso:: The :ref:`key_encryption` section for an in-depth review of the key encryption.\n\nWhen initializing a repository with one of the \"keyfile\" encryption modes,\nBorg creates an associated key file in the keys subdirectory of the borg config\ndirectory (see :ref:`env_vars` for platform-specific default paths).\n\nThe same key is also used in the \"repokey\" modes, which store it in the repository.\n\nThe internal data structure is as follows:\n\nversion\n currently always an integer, 2\n\nrepository_id\n the ``id`` field in the ``config`` ``INI`` file of the repository.\n\ncrypt_key\n the initial key material used for the AEAD crypto (512 bits)\n\nid_key\n the key used to MAC the plaintext chunk data to compute the chunk's id\n\nchunk_seed\n the seed for the buzhash chunking table (signed 32 bit integer)\n\nThese fields are packed using msgpack_. The utf-8 encoded passphrase\nis processed with argon2_ to derive a 256 bit key encryption key (KEK).\n\nThen the KEK is used to encrypt and authenticate the packed data using\nthe chacha20-poly1305 AEAD cipher.\n\nThe result is stored in a another msgpack_ formatted as follows:\n\nversion\n currently always an integer, 1\n\nsalt\n random 256 bits salt used to process the passphrase\n\nargon2_*\n some parameters for the argon2 kdf\n\nalgorithm\n the algorithms used to process the passphrase\n (currently the string ``argon2 chacha20-poly1305``)\n\ndata\n The encrypted, packed fields.\n\nThe resulting msgpack_ is then encoded using base64 and written to the\nkey file, wrapped using the standard ``textwrap`` module with a header.\nThe header is a single line with a MAGIC string, a space and a hexadecimal\nrepresentation of the repository id.\n\n.. _data-compression:\n\nCompression\n-----------\n\nBorg supports the following compression methods, each identified by a ctype value\nin the range between 0 and 255 (and augmented by a clevel 0..255 value for the\ncompression level):\n\n- none (no compression, pass through data 1:1), identified by 0x00\n- lz4 (low compression, but super fast), identified by 0x01\n- zstd (level 1-22 offering a wide range: level 1 is lower compression and high\n speed, level 22 is higher compression and lower speed) - identified by 0x03\n- zlib (level 0-9, level 0 is no compression [but still adding zlib overhead],\n level 1 is low, level 9 is high compression), identified by 0x05\n- lzma (level 0-9, level 0 is low, level 9 is high compression), identified\n by 0x02.\n\nThe type byte is followed by a byte indicating the compression level.\n\nSpeed: none > lz4 > zlib > lzma, lz4 > zstd\nCompression: lzma > zlib > lz4 > none, zstd > lz4\n\nBe careful, higher compression levels might use a lot of resources (CPU/memory).\n\nThe overall speed of course also depends on the speed of your target storage.\nIf that is slow, using a higher compression level might yield better overall\nperformance. You need to experiment a bit. Maybe just watch your CPU load, if\nthat is relatively low, increase compression until 1 core is 70-100% loaded.\n\nEven if your target storage is rather fast, you might see interesting effects:\nwhile doing no compression at all (none) is a operation that takes no time, it\nlikely will need to store more data to the storage compared to using lz4.\nThe time needed to transfer and store the additional data might be much more\nthan if you had used lz4 (which is super fast, but still might compress your\ndata about 2:1). This is assuming your data is compressible (if you back up\nalready compressed data, trying to compress them at backup time is usually\npointless).\n\nCompression is applied after deduplication, thus using different compression\nmethods in one repo does not influence deduplication.\n\nSee ``borg create --help`` about how to specify the compression level and its default.\n\nLock files (fslocking)\n----------------------\n\nBorg uses filesystem locks to get (exclusive or shared) access to the cache.\n\nThe locking system is based on renaming a temporary directory\nto `lock.exclusive` (for\nexclusive locks). Inside this directory, there is a file indicating\nhostname, process id and thread id of the lock holder.\n\nThere is also a json file `lock.roster` that keeps a directory of all shared\nand exclusive lockers.\n\nIf the process is able to rename a temporary directory (with the\nhost/process/thread identifier prepared inside it) in the resource directory\nto `lock.exclusive`, it has the lock for it. If renaming fails\n(because this directory already exists and its host/process/thread identifier\ndenotes a thread on the host which is still alive), lock acquisition fails.\n\nThe cache lock is usually in `~/.cache/borg/REPOID/lock.*`.\n\nLocks (storelocking)\n--------------------\n\nTo implement locking based on ``borgstore``, borg stores objects below locks/.\n\nThe objects contain:\n\n- a timestamp when lock was created (or refreshed)\n- host / process / thread information about lock owner\n- lock type: exclusive or shared\n\nUsing that information, borg implements:\n\n- lock auto-expiry: if a lock is old and has not been refreshed in time,\n it will be automatically ignored and deleted. the primary purpose of this\n is to get rid of stale locks by borg processes on other machines.\n- lock auto-removal if the owner process is dead. the primary purpose of this\n is to quickly get rid of stale locks by borg processes on the same machine.\n\nBreaking the locks\n------------------\n\nIn case you run into troubles with the locks, you can use the ``borg break-lock``\ncommand after you first have made sure that no Borg process is\nrunning on any machine that accesses this resource. Be very careful, the cache\nor repository might get damaged if multiple processes use it at the same time.\n\nIf there is an issue just with the repository lock, it will usually resolve\nautomatically (see above), just retry later.\n\n\nChecksumming data structures\n----------------------------\n\nAs detailed in the previous sections, Borg generates and stores various files\ncontaining important meta data, such as the files cache.\n\nData corruption in the files cache could create incorrect archives, e.g. due\nto wrong object IDs or sizes in the files cache.\n\nTherefore, Borg calculates checksums when writing these files and tests checksums\nwhen reading them. Checksums are generally 64-bit XXH64 hashes.\nThe canonical xxHash representation is used, i.e. big-endian.\nChecksums are stored as hexadecimal ASCII strings.\n\nFor compatibility, checksums are not required and absent checksums do not trigger errors.\nThe mechanisms have been designed to avoid false-positives when various Borg\nversions are used alternately on the same repositories.\n\nChecksums are a data safety mechanism. They are not a security mechanism.\n\n.. rubric:: Choice of algorithm\n\nXXH64 has been chosen for its high speed on all platforms, which avoids performance\ndegradation in CPU-limited parts (e.g. cache synchronization).\nUnlike CRC32, it neither requires hardware support (crc32c or CLMUL)\nnor vectorized code nor large, cache-unfriendly lookup tables to achieve good performance.\nThis simplifies deployment of it considerably (cf. src/borg/algorithms/crc32...).\n\nFurther, XXH64 is a non-linear hash function and thus has a \"more or less\" good\nchance to detect larger burst errors, unlike linear CRCs where the probability\nof detection decreases with error size.\n\nThe 64-bit checksum length is considered sufficient for the file sizes typically\nchecksummed (individual files up to a few GB, usually less).\nxxHash was expressly designed for data blocks of these sizes.\n\nLower layer — file_integrity\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThere is a lower layer (borg.crypto.file_integrity.IntegrityCheckedFile)\nwrapping a file-like object, performing streaming calculation and comparison\nof checksums.\nChecksum errors are signalled by raising an exception at the earliest possible\nmoment (borg.crypto.file_integrity.FileIntegrityError).\n\n.. rubric:: Calculating checksums\n\nBefore feeding the checksum algorithm any data, the file name (i.e. without any path)\nis mixed into the checksum, since the name encodes the context of the data for Borg.\n\nThe various indices used by Borg have separate header and main data parts.\nIntegrityCheckedFile allows borg to checksum them independently, which avoids\neven reading the data when the header is corrupted. When a part is signalled,\nthe length of the part name is mixed into the checksum state first (encoded\nas an ASCII string via `%10d` printf format), then the name of the part\nis mixed in as an UTF-8 string. Lastly, the current position (length)\nin the file is mixed in as well.\n\nThe checksum state is not reset at part boundaries.\n\nA final checksum is always calculated in the same way as the parts described above,\nafter seeking to the end of the file. The final checksum cannot prevent code\nfrom processing corrupted data during reading, however, it prevents use of the\ncorrupted data.\n\n.. rubric:: Serializing checksums\n\nAll checksums are compiled into a simple JSON structure called *integrity data*:\n\n.. code-block:: json\n\n {\n \"algorithm\": \"XXH64\",\n \"digests\": {\n \"HashHeader\": \"eab6802590ba39e3\",\n \"final\": \"e2a7f132fc2e8b24\"\n }\n }\n\nThe *algorithm* key notes the used algorithm. When reading, integrity data containing\nan unknown algorithm is not inspected further.\n\nThe *digests* key contains a mapping of part names to their digests.\n\nIntegrity data is generally stored by the upper layers, introduced below. An exception\nis the DetachedIntegrityCheckedFile, which automatically writes and reads it from\na \".integrity\" file next to the data file.\n\nUpper layer\n~~~~~~~~~~~\n\n.. rubric:: Main cache files: chunks and files cache\n\nThe integrity data of the ``files`` cache is stored in the cache ``config``.\n\nThe ``[integrity]`` section is used:\n\n.. code-block:: none\n\n [cache]\n version = 1\n repository = 3c4...e59\n manifest = 10e...21c\n timestamp = 2017-06-01T21:31:39.699514\n key_type = 2\n previous_location = /path/to/repo\n\n [integrity]\n manifest = 10e...21c\n files = {\"algorithm\": \"XXH64\", \"digests\": {\"HashHeader\": \"eab...39e3\", \"final\": \"e2a...b24\"}}\n\nThe manifest ID is duplicated in the integrity section due to the way all Borg\nversions handle the config file. Instead of creating a \"new\" config file from\nan internal representation containing only the data understood by Borg,\nthe config file is read in entirety (using the Python ConfigParser) and modified.\nThis preserves all sections and values not understood by the Borg version\nmodifying it.\n\nThus, if an older versions uses a cache with integrity data, it would preserve\nthe integrity section and its contents. If a integrity-aware Borg version\nwould read this cache, it would incorrectly report checksum errors, since\nthe older version did not update the checksums.\n\nHowever, by duplicating the manifest ID in the integrity section, it is\neasy to tell whether the checksums concern the current state of the cache.\n\nIntegrity errors are fatal in these files, terminating the program,\nand are not automatically corrected at this time.\n\n\nHardLinkManager and the hlid concept\n------------------------------------\n\nDealing with hard links needs some extra care, implemented in borg within the HardLinkManager\nclass:\n\n- At archive creation time, fs items with st_nlink > 1 indicate that they are a member of\n a group of hardlinks all pointing to the same inode. For such fs items, the archived item\n includes a hlid attribute (hardlink id), which is computed like H(st_dev, st_ino). Thus,\n if archived items have the same hlid value, they pointed to the same inode and form a\n group of hardlinks. Besides that, nothing special is done for any member of the group\n of hardlinks, meaning that e.g. for regular files, each archived item will have a\n chunks list.\n- At extraction time, the presence of a hlid attribute indicates that there might be more\n hardlinks coming, pointing to the same content (inode), thus borg will remember the \"hlid\n to extracted path\" mapping, so it will know the correct path for extracting (hardlinking)\n the next hardlink of that group / with the same hlid.\n- This symmetric approach (each item has all the information, e.g. the chunks list)\n simplifies dealing with such items a lot, especially for partial extraction, for the\n FUSE filesystem, etc.\n- This is different from the asymmetric approach of old borg versions (< 2.0) and also from\n tar which have the concept of a main item (first hardlink, has the content) and content-less\n secondary items with by-name back references for each subsequent hardlink, causing lots\n of complications when dealing with them.\n" }, { "path": "docs/internals/frontends.rst", "content": ".. include:: ../global.rst.inc\n.. highlight:: none\n\n.. _json_output:\n\nAll about JSON: How to develop frontends\n========================================\n\nBorg does not have a public API on the Python level. That does not keep you from writing :code:`import borg`,\nbut does mean that there are no release-to-release guarantees on what you might find in that package, not\neven for point releases (1.1.x), and there is no documentation beyond the code and the internals documents.\n\nBorg does on the other hand provide an API on a command-line level. In other words, a frontend should\n(for example) create a backup archive by invoking :ref:`borg_create`, provide command-line parameters/options\nas needed, and parse JSON output from Borg.\n\nImportant: JSON output is expected to be UTF-8, but currently borg depends on the locale being configured\nfor that (must be a UTF-8 locale and *not* \"C\" or \"ascii\"), so that Python will choose to encode to UTF-8.\nThe same applies to any inputs read by borg, they are expected to be UTF-8 encoded also.\n\nOn POSIX systems, you can usually set environment vars to choose a UTF-8 locale:\n\n::\n\n export LANG=en_US.UTF-8\n export LC_CTYPE=en_US.UTF-8\n\n\nAnother way to get Python's stdin/stdout/stderr streams to use UTF-8 encoding (without having\na UTF-8 locale / LANG / LC_CTYPE) is:\n\n::\n\n export PYTHONIOENCODING=utf-8\n\n\nSee :issue:`2273` for more details.\n\n\nDealing with non-unicode byte sequences and JSON limitations\n------------------------------------------------------------\n\nPaths on POSIX systems can have arbitrary bytes in them (except 0x00 which is used as string terminator in C).\n\nNowadays, UTF-8 encoded paths (which decode to valid unicode) are the usual thing, but a lot of systems\nstill have paths from the past, when other, non-unicode codings were used. Especially old Samba shares often\nhave wild mixtures of misc. encodings, sometimes even very broken stuff.\n\nborg deals with such non-unicode paths (\"with funny/broken characters\") by decoding such byte sequences using\nUTF-8 coding and \"surrogateescape\" error handling mode, which maps invalid bytes to special unicode code points\n(surrogate escapes). When encoding such a unicode string back to a byte sequence, the original byte sequence\nwill be reproduced exactly.\n\nJSON should only contain valid unicode text without any surrogate escapes, so we can't just directly have a\nsurrogate-escaped path in JSON (\"path\" is only one example, this also affects other text-like content).\n\nBorg deals with this situation like this (since borg 2.0):\n\nFor a valid unicode path (no surrogate escapes), the JSON will only have \"path\": path.\n\nFor a non-unicode path (with surrogate escapes), the JSON will have 2 entries:\n\n- \"path\": path_approximation (pure valid unicode, all invalid bytes will show up as \"?\")\n- \"path_b64\": path_bytes_base64_encoded (if you decode the base64, you get the original path byte string)\n\nJSON users need to pick whatever suits their needs best. The suggested procedure (shown for \"path\") is:\n\n- check if there is a \"path_b64\" key.\n- if it is there, you will know that the original bytes path did not cleanly UTF-8-decode into unicode (has\n some invalid bytes) and that the string given by the \"path\" key is only an approximation, but not the precise\n path. if you need precision, you must base64-decode the value of \"path_b64\" and deal with the arbitrary byte\n string you'll get. if an approximation is fine, use the value of the \"path\" key.\n- if it is not there, the value of the \"path\" key is all you need (the original bytes path is its UTF-8 encoding).\n\n\nLogging\n-------\n\nEspecially for graphical frontends it is important to be able to convey and reformat progress information\nin meaningful ways. The ``--log-json`` option turns the stderr stream of Borg into a stream of JSON lines,\nwhere each line is a JSON object. The *type* key of the object determines its other contents.\n\n.. warning:: JSON logging requires successful argument parsing. Even with ``--log-json`` specified, a\n parsing error will be printed in plain text, because logging set-up happens after all arguments are\n parsed.\n\nThe following types are in use. Progress information is governed by the usual rules for progress information,\nit is not produced unless ``--progress`` is specified.\n\narchive_progress\n Output during operations creating archives (:ref:`borg_create` and :ref:`borg_recreate`).\n The following keys exist, each represents the current progress.\n\n original_size\n Original size of data processed so far (before compression and deduplication, may be empty/absent)\n compressed_size\n Compressed size (may be empty/absent)\n deduplicated_size\n Deduplicated size (may be empty/absent)\n nfiles\n Number of (regular) files processed so far (may be empty/absent)\n path\n Current path (may be empty/absent)\n time\n Unix timestamp (float)\n finished\n boolean indicating whether the operation has finished, only the last object for an *operation*\n can have this property set to *true*.\n\nprogress_message\n A message-based progress information with no concrete progress information, just a message\n saying what is currently being worked on.\n\n operation\n unique, opaque integer ID of the operation\n :ref:`msgid `\n Message ID of the operation (may be *null*)\n finished\n boolean indicating whether the operation has finished, only the last object for an *operation*\n can have this property set to *true*.\n message\n current progress message (may be empty/absent)\n time\n Unix timestamp (float)\n\nprogress_percent\n Absolute progress information with defined end/total and current value.\n\n operation\n unique, opaque integer ID of the operation\n :ref:`msgid `\n Message ID of the operation (may be *null*)\n finished\n boolean indicating whether the operation has finished, only the last object for an *operation*\n can have this property set to *true*.\n message\n A formatted progress message, this will include the percentage and perhaps other information\n (absent for finished == true)\n current\n Current value (always less-or-equal to *total*, absent for finished == true)\n info\n Array that describes the current item, may be *null*, contents depend on *msgid*\n (absent for finished == true)\n total\n Total value (absent for finished == true)\n time\n Unix timestamp (float)\n\nfile_status\n This is only output by :ref:`borg_create` and :ref:`borg_recreate` if ``--list`` is specified. The usual\n rules for the file listing applies, including the ``--filter`` option.\n\n status\n Single-character status as for regular list output\n path\n Path of the file system object\n\nlog_message\n Any regular log output invokes this type. Regular log options and filtering applies to these as well.\n\n time\n Unix timestamp (float)\n levelname\n Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL\n name\n Name of the emitting entity\n message\n Formatted log message\n :ref:`msgid `\n Message ID, may be *null* or absent\n\nSee Prompts_ for the types used by prompts.\n\n.. rubric:: Examples (reformatted, each object would be on exactly one line)\n.. highlight:: json\n\n:ref:`borg_extract` progress::\n\n {\"message\": \"100.0% Extracting: src/borgbackup.egg-info/entry_points.txt\",\n \"current\": 13000228, \"total\": 13004993, \"info\": [\"src/borgbackup.egg-info/entry_points.txt\"],\n \"operation\": 1, \"msgid\": \"extract\", \"type\": \"progress_percent\", \"finished\": false}\n {\"message\": \"100.0% Extracting: src/borgbackup.egg-info/SOURCES.txt\",\n \"current\": 13004993, \"total\": 13004993, \"info\": [\"src/borgbackup.egg-info/SOURCES.txt\"],\n \"operation\": 1, \"msgid\": \"extract\", \"type\": \"progress_percent\", \"finished\": false}\n {\"operation\": 1, \"msgid\": \"extract\", \"type\": \"progress_percent\", \"finished\": true}\n\n:ref:`borg_create` file listing with progress::\n\n {\"original_size\": 0, \"compressed_size\": 0, \"deduplicated_size\": 0, \"nfiles\": 0, \"type\": \"archive_progress\", \"path\": \"src\"}\n {\"type\": \"file_status\", \"status\": \"U\", \"path\": \"src/borgbackup.egg-info/entry_points.txt\"}\n {\"type\": \"file_status\", \"status\": \"U\", \"path\": \"src/borgbackup.egg-info/SOURCES.txt\"}\n {\"type\": \"file_status\", \"status\": \"d\", \"path\": \"src/borgbackup.egg-info\"}\n {\"type\": \"file_status\", \"status\": \"d\", \"path\": \"src\"}\n {\"original_size\": 13176040, \"compressed_size\": 11386863, \"deduplicated_size\": 503, \"nfiles\": 277, \"type\": \"archive_progress\", \"path\": \"\"}\n\nInternal transaction progress::\n\n {\"message\": \"Saving files cache\", \"operation\": 2, \"msgid\": \"cache.commit\", \"type\": \"progress_message\", \"finished\": false}\n {\"message\": \"Saving cache config\", \"operation\": 2, \"msgid\": \"cache.commit\", \"type\": \"progress_message\", \"finished\": false}\n {\"message\": \"Saving chunks cache\", \"operation\": 2, \"msgid\": \"cache.commit\", \"type\": \"progress_message\", \"finished\": false}\n {\"operation\": 2, \"msgid\": \"cache.commit\", \"type\": \"progress_message\", \"finished\": true}\n\nA debug log message::\n\n {\"message\": \"35 self tests completed in 0.08 seconds\",\n \"type\": \"log_message\", \"created\": 1488278449.5575905, \"levelname\": \"DEBUG\", \"name\": \"borg.archiver\"}\n\nPrompts\n-------\n\nPrompts assume a JSON form as well when the ``--log-json`` option is specified. Responses\nare still read verbatim from *stdin*, while prompts are JSON messages printed to *stderr*,\njust like log messages.\n\nPrompts use the *question_prompt* and *question_prompt_retry* types for the prompt itself,\nand *question_invalid_answer*, *question_accepted_default*, *question_accepted_true*,\n*question_accepted_false* and *question_env_answer* types for information about\nprompt processing.\n\nThe *message* property contains the same string displayed regularly in the same situation,\nwhile the *msgid* property may contain a msgid_, typically the name of the\nenvironment variable that can be used to override the prompt. It is the same for all JSON\nmessages pertaining to the same prompt.\n\n.. rubric:: Examples (reformatted, each object would be on exactly one line)\n.. highlight:: none\n\nProviding an invalid answer::\n\n {\"type\": \"question_prompt\", \"msgid\": \"BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\",\n \"message\": \"... Type 'YES' if you understand this and want to continue: \"}\n incorrect answer # input on stdin\n {\"type\": \"question_invalid_answer\", \"msgid\": \"BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\", \"is_prompt\": false,\n \"message\": \"Invalid answer, aborting.\"}\n\nProviding a false (negative) answer::\n\n {\"type\": \"question_prompt\", \"msgid\": \"BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\",\n \"message\": \"... Type 'YES' if you understand this and want to continue: \"}\n NO # input on stdin\n {\"type\": \"question_accepted_false\", \"msgid\": \"BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\",\n \"message\": \"Aborting.\", \"is_prompt\": false}\n\nProviding a true (affirmative) answer::\n\n {\"type\": \"question_prompt\", \"msgid\": \"BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\",\n \"message\": \"... Type 'YES' if you understand this and want to continue: \"}\n YES # input on stdin\n # no further output, just like the prompt without --log-json\n\nPassphrase prompts\n------------------\n\nPassphrase prompts should be handled differently. Use the environment variables *BORG_PASSPHRASE*\nand *BORG_NEW_PASSPHRASE* (see :ref:`env_vars` for reference) to pass passphrases to Borg, don't\nuse the interactive passphrase prompts.\n\nWhen setting a new passphrase (:ref:`borg_repo-create`, :ref:`borg_key_change-passphrase`) normally\nBorg prompts whether it should display the passphrase. This can be suppressed by setting\nthe environment variable *BORG_DISPLAY_PASSPHRASE* to *no*.\n\nWhen \"confronted\" with an unknown repository, where the application does not know whether\nthe repository is encrypted, the following algorithm can be followed to detect encryption:\n\n1. Set *BORG_PASSPHRASE* to gibberish (for example a freshly generated UUID4, which cannot\n possibly be the passphrase)\n2. Invoke ``borg list repository ...``\n3. If this fails, due the repository being encrypted and the passphrase obviously being\n wrong, you'll get an error with the *PassphraseWrong* msgid.\n\n The repository is encrypted, for further access the application will need the passphrase.\n\n4. If this does not fail, then the repository is not encrypted.\n\nStandard output\n---------------\n\n*stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create`\nand :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object.\n\nSome commands, like :ref:`borg_list` and :ref:`borg_diff`, can produce *a lot* of JSON. Since many JSON implementations\ndon't support a streaming mode of operation, which is pretty much required to deal with this amount of JSON, these\ncommands implement a ``--json-lines`` option which generates output in the `JSON lines `_ format,\nwhich is simply a number of JSON objects separated by new lines.\n\nDates are formatted according to ISO 8601 in local time. No explicit time zone is specified *at this time*\n(subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S.%f',\ne.g. ``2017-08-07T12:27:20.123456``.\n\nThe root object of '--json' output will contain at least a *repository* key with an object containing:\n\nid\n The ID of the repository, normally 64 hex characters\nlocation\n Canonicalized repository path, thus this may be different from what is specified on the command line\nlast_modified\n Date when the repository was last modified by the Borg client\n\nThe *encryption* key, if present, contains:\n\nmode\n Textual encryption mode name (same as :ref:`borg_repo-create` ``--encryption`` names)\nkeyfile\n Path to the local key file used for access. Depending on *mode* this key may be absent.\n\nThe *cache* key, if present, contains:\n\npath\n Path to the local repository cache\nstats\n Object containing cache stats:\n\n total_chunks\n Number of chunks\n total_unique_chunks\n Number of unique chunks\n total_size\n Total uncompressed size of all chunks multiplied with their reference counts\n unique_size\n Uncompressed size of all chunks\n\n.. highlight: json\n\nExample *borg info* output::\n\n {\n \"cache\": {\n \"path\": \"/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"stats\": {\n \"total_chunks\": 511533,\n \"total_size\": 22635749792,\n \"total_unique_chunks\": 54892,\n \"unique_size\": 2449675468\n }\n },\n \"encryption\": {\n \"mode\": \"repokey\"\n },\n \"repository\": {\n \"id\": \"0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"last_modified\": \"2017-08-07T12:27:20.789123\",\n \"location\": \"/home/user/testrepo\"\n },\n \"security_dir\": \"/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"archives\": []\n }\n\nArchive formats\n+++++++++++++++\n\n:ref:`borg_info` uses an extended format for archives, which is more expensive to retrieve, while\n:ref:`borg_list` uses a simpler format that is faster to retrieve. Either return archives in an\narray under the *archives* key, while :ref:`borg_create` returns a single archive object under the\n*archive* key.\n\nBoth formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID,\nand the *start* key with the start timestamp.\n\n*borg info* and *borg create* further have:\n\nend\n End timestamp\nduration\n Duration in seconds between start and end in seconds (float)\nstats\n Archive statistics (freshly calculated, this is what makes \"info\" more expensive)\n\n original_size\n Size of files and metadata before compression\n compressed_size\n Size after compression\n deduplicated_size\n Deduplicated size (against the current repository, not when the archive was created)\n nfiles\n Number of regular files in the archive\ncommand_line\n Array of strings of the command line that created the archive\n\n The note about paths from above applies here as well.\nchunker_params\n The chunker parameters the archive has been created with.\n\n:ref:`borg_info` further has:\n\nhostname\n Hostname of the creating host\nusername\n Name of the creating user\ncomment\n Archive comment, if any\n\nSome keys/values are more expensive to compute than others (e.g. because it requires opening the archive,\nnot just the manifest). To optimize for speed, `borg list repo` does not determine these values except\nwhen they are requested. The `--format` option is used for that (for normal mode as well as for `--json`\nmode), so, to have the comment included in the json output, you will need:\n\n::\n\n borg list repo --format \"{name}{comment}\" --json`\n\n\nExample of a simple archive listing (``borg list --last 1 --json``)::\n\n {\n \"archives\": [\n {\n \"id\": \"80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a\",\n \"name\": \"host-system-backup-2017-02-27\",\n \"start\": \"2017-08-07T12:27:20.789123\"\n }\n ],\n \"encryption\": {\n \"mode\": \"repokey\"\n },\n \"repository\": {\n \"id\": \"0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"last_modified\": \"2017-08-07T12:27:20.789123\",\n \"location\": \"/home/user/repository\"\n }\n }\n\nThe same archive with more information (``borg info --last 1 --json``)::\n\n {\n \"archives\": [\n {\n \"chunker_params\": [\n \"buzhash\",\n 13,\n 23,\n 16,\n 4095\n ],\n \"command_line\": [\n \"/home/user/.local/bin/borg\",\n \"create\",\n \"/home/user/repository\",\n \"...\"\n ],\n \"comment\": \"\",\n \"duration\": 5.641542,\n \"end\": \"2017-02-27T12:27:20.789123\",\n \"hostname\": \"host\",\n \"id\": \"80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a\",\n \"name\": \"host-system-backup-2017-02-27\",\n \"start\": \"2017-02-27T12:27:20.789123\",\n \"stats\": {\n \"compressed_size\": 1880961894,\n \"deduplicated_size\": 2791,\n \"nfiles\": 53669,\n \"original_size\": 2400471280\n },\n \"username\": \"user\"\n }\n ],\n \"cache\": {\n \"path\": \"/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"stats\": {\n \"total_chunks\": 511533,\n \"total_size\": 22635749792,\n \"total_unique_chunks\": 54892,\n \"unique_size\": 2449675468\n }\n },\n \"encryption\": {\n \"mode\": \"repokey\"\n },\n \"repository\": {\n \"id\": \"0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23\",\n \"last_modified\": \"2017-08-07T12:27:20.789123\",\n \"location\": \"/home/user/repository\"\n }\n }\n\nFile listings\n+++++++++++++\n\nEach archive item (file, directory, ...) is described by one object in the :ref:`borg_list` output.\nRefer to the *borg list* documentation for the available keys and their meaning.\n\nExample (excerpt) of ``borg list --json-lines``::\n\n {\"type\": \"d\", \"mode\": \"drwxr-xr-x\", \"user\": \"user\", \"group\": \"user\", \"uid\": 1000, \"gid\": 1000, \"path\": \"linux\", \"target\": \"\", \"flags\": null, \"mtime\": \"2017-02-27T12:27:20.023407\", \"size\": 0}\n {\"type\": \"d\", \"mode\": \"drwxr-xr-x\", \"user\": \"user\", \"group\": \"user\", \"uid\": 1000, \"gid\": 1000, \"path\": \"linux/baz\", \"target\": \"\", \"flags\": null, \"mtime\": \"2017-02-27T12:27:20.585407\", \"size\": 0}\n\n\nArchive Differencing\n++++++++++++++++++++\n\nEach archive difference item (file contents, user/group/mode) output by :ref:`borg_diff` is represented by an *ItemDiff* object.\nThe properties of an *ItemDiff* object are:\n\npath:\n The filename/path of the *Item* (file, directory, symlink).\n\nchanges:\n A list of *Change* objects describing the changes made to the item in the two archives. For example,\n there will be two changes if the contents of a file are changed, and its ownership are changed.\n\nThe *Change* object can contain a number of properties depending on the type of change that occurred.\nIf a 'property' is not required for the type of change, it is not output.\nThe possible properties of a *Change* object are:\n\ntype:\n The **type** property is always present. It identifies the type of change and will be one of these values:\n\n - *modified* - file contents changed.\n - *added* - the file was added.\n - *removed* - the file was removed.\n - *added directory* - the directory was added.\n - *removed directory* - the directory was removed.\n - *added link* - the symlink was added.\n - *removed link* - the symlink was removed.\n - *changed link* - the symlink target was changed.\n - *mode* - the file/directory/link mode was changed. Note - this could indicate a change from a\n file/directory/link type to a different type (file/directory/link), such as -- a file is deleted and replaced\n with a directory of the same name.\n - *owner* - user and/or group ownership changed.\n\nsize:\n If **type** == '*added*' or '*removed*', then **size** provides the size of the added or removed file.\n\nadded:\n If **type** == '*modified*' and chunk ids can be compared, then **added** and **removed** indicate the amount\n of data 'added' and 'removed'. If chunk ids can not be compared, then **added** and **removed** properties are\n not provided and the only information available is that the file contents were modified.\n\nremoved:\n See **added** property.\n\nold_mode:\n If **type** == '*mode*', then **old_mode** and **new_mode** provide the mode and permissions changes.\n\nnew_mode:\n See **old_mode** property.\n\nold_user:\n If **type** == '*owner*', then **old_user**, **new_user**, **old_group** and **new_group** provide the user\n and group ownership changes.\n\nold_group:\n See **old_user** property.\n\nnew_user:\n See **old_user** property.\n\nnew_group:\n See **old_user** property.\n\n\nExample (excerpt) of ``borg diff --json-lines``::\n\n {\"path\": \"file1\", \"changes\": [{\"path\": \"file1\", \"changes\": [{\"type\": \"modified\", \"added\": 17, \"removed\": 5}, {\"type\": \"mode\", \"old_mode\": \"-rw-r--r--\", \"new_mode\": \"-rwxr-xr-x\"}]}]}\n {\"path\": \"file2\", \"changes\": [{\"type\": \"modified\", \"added\": 135, \"removed\": 252}]}\n {\"path\": \"file4\", \"changes\": [{\"type\": \"added\", \"size\": 0}]}\n {\"path\": \"file3\", \"changes\": [{\"type\": \"removed\", \"size\": 0}]}\n\n\n.. _msgid:\n\nMessage IDs\n-----------\n\nMessage IDs are strings that essentially give a log message or operation a name, without actually using the\nfull text, since texts change more frequently. Message IDs are unambiguous and reduce the need to parse\nlog messages.\n\nAssigned message IDs and related error RCs (exit codes) are:\n\n.. See scripts/errorlist.py; this is slightly edited.\n\nErrors\n Error rc: 2 traceback: no\n Error: {}\n ErrorWithTraceback rc: 2 traceback: yes\n Error: {}\n\n Buffer.MemoryLimitExceeded rc: 2 traceback: no\n Requested buffer size {} is above the limit of {}.\n EfficientCollectionQueue.SizeUnderflow rc: 2 traceback: no\n Could not pop_front first {} elements, collection only has {} elements..\n RTError rc: 2 traceback: no\n Runtime Error: {}\n\n CancelledByUser rc: 3 traceback: no\n Cancelled by user.\n\n CommandError rc: 4 traceback: no\n Command Error: {}\n PlaceholderError rc: 5 traceback: no\n Formatting Error: \"{}\".format({}): {}({})\n InvalidPlaceholder rc: 6 traceback: no\n Invalid placeholder \"{}\" in string: {}\n\n Repository.AlreadyExists rc: 10 traceback: no\n A repository already exists at {}.\n Repository.CheckNeeded rc: 12 traceback: yes\n Inconsistency detected. Please run \"borg check {}\".\n Repository.DoesNotExist rc: 13 traceback: no\n Repository {} does not exist.\n Repository.InsufficientFreeSpaceError rc: 14 traceback: no\n Insufficient free space to complete transaction (required: {}, available: {}).\n Repository.InvalidRepository rc: 15 traceback: no\n {} is not a valid repository. Check repo config.\n Repository.InvalidRepositoryConfig rc: 16 traceback: no\n {} does not have a valid configuration. Check repo config [{}].\n Repository.ObjectNotFound rc: 17 traceback: yes\n Object with key {} not found in repository {}.\n Repository.ParentPathDoesNotExist rc: 18 traceback: no\n The parent path of the repo directory [{}] does not exist.\n Repository.PathAlreadyExists rc: 19 traceback: no\n There is already something at {}.\n Repository.PathPermissionDenied rc: 21 traceback: no\n Permission denied to {}.\n\n MandatoryFeatureUnsupported rc: 25 traceback: no\n Unsupported repository feature(s) {}. A newer version of borg is required to access this repository.\n NoManifestError rc: 26 traceback: no\n Repository has no manifest.\n UnsupportedManifestError rc: 27 traceback: no\n Unsupported manifest envelope. A newer version is required to access this repository.\n\n Archive.AlreadyExists rc: 30 traceback: no\n Archive {} already exists\n Archive.DoesNotExist rc: 31 traceback: no\n Archive {} does not exist\n Archive.IncompatibleFilesystemEncodingError rc: 32 traceback: no\n Failed to encode filename \"{}\" into file system encoding \"{}\". Consider configuring the LANG environment variable.\n\n KeyfileInvalidError rc: 40 traceback: no\n Invalid key data for repository {} found in {}.\n KeyfileMismatchError rc: 41 traceback: no\n Mismatch between repository {} and key file {}.\n KeyfileNotFoundError rc: 42 traceback: no\n No key file for repository {} found in {}.\n NotABorgKeyFile rc: 43 traceback: no\n This file is not a borg key backup, aborting.\n RepoKeyNotFoundError rc: 44 traceback: no\n No key entry found in the config of repository {}.\n RepoIdMismatch rc: 45 traceback: no\n This key backup seems to be for a different backup repository, aborting.\n UnencryptedRepo rc: 46 traceback: no\n Key management not available for unencrypted repositories.\n UnknownKeyType rc: 47 traceback: no\n Key type {0} is unknown.\n UnsupportedPayloadError rc: 48 traceback: no\n Unsupported payload type {}. A newer version is required to access this repository.\n UnsupportedKeyFormatError rc: 49 traceback:no\n Your borg key is stored in an unsupported format. Try using a newer version of borg.\n\n\n NoPassphraseFailure rc: 50 traceback: no\n can not acquire a passphrase: {}\n PasscommandFailure rc: 51 traceback: no\n passcommand supplied in BORG_PASSCOMMAND failed: {}\n PassphraseWrong rc: 52 traceback: no\n passphrase supplied in BORG_PASSPHRASE, by BORG_PASSCOMMAND or via BORG_PASSPHRASE_FD is incorrect.\n PasswordRetriesExceeded rc: 53 traceback: no\n exceeded the maximum password retries\n\n Cache.CacheInitAbortedError rc: 60 traceback: no\n Cache initialization aborted\n Cache.EncryptionMethodMismatch rc: 61 traceback: no\n Repository encryption method changed since last access, refusing to continue\n Cache.RepositoryAccessAborted rc: 62 traceback: no\n Repository access aborted\n Cache.RepositoryIDNotUnique rc: 63 traceback: no\n Cache is newer than repository - do you have multiple, independently updated repos with same ID?\n Cache.RepositoryReplay rc: 64 traceback: no\n Cache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID)\n\n LockError rc: 70 traceback: no\n Failed to acquire the lock {}.\n LockErrorT rc: 71 traceback: yes\n Failed to acquire the lock {}.\n LockFailed rc: 72 traceback: yes\n Failed to create/acquire the lock {} ({}).\n LockTimeout rc: 73 traceback: no\n Failed to create/acquire the lock {} (timeout).\n NotLocked rc: 74 traceback: yes\n Failed to release the lock {} (was not locked).\n NotMyLock rc: 75 traceback: yes\n Failed to release the lock {} (was/is locked, but not by me).\n\n ConnectionClosed rc: 80 traceback: no\n Connection closed by remote host\n ConnectionClosedWithHint rc: 81 traceback: no\n Connection closed by remote host. {}\n InvalidRPCMethod rc: 82 traceback: no\n RPC method {} is not valid\n PathNotAllowed rc: 83 traceback: no\n Repository path not allowed: {}\n RemoteRepository.RPCServerOutdated rc: 84 traceback: no\n Borg server is too old for {}. Required version {}\n UnexpectedRPCDataFormatFromClient rc: 85 traceback: no\n Borg {}: Got unexpected RPC data format from client.\n UnexpectedRPCDataFormatFromServer rc: 86 traceback: no\n Got unexpected RPC data format from server:\n {}\n ConnectionBrokenWithHint rc: 87 traceback: no\n Connection to remote host is broken. {}\n\n IntegrityError rc: 90 traceback: yes\n Data integrity error: {}\n FileIntegrityError rc: 91 traceback: yes\n File failed integrity check: {}\n DecompressionError rc: 92 traceback: yes\n Decompression error: {}\n\n\nWarnings\n BorgWarning rc: 1\n Warning: {}\n BackupWarning rc: 1\n {}: {}\n\n FileChangedWarning rc: 100\n {}: file changed while we backed it up\n IncludePatternNeverMatchedWarning rc: 101\n Include pattern '{}' never matched.\n BackupError rc: 102\n {}: backup error\n BackupRaceConditionError rc: 103\n {}: file type or inode changed while we backed it up (race condition, skipped file)\n BackupOSError rc: 104\n {}: {}\n BackupPermissionError rc: 105\n {}: {}\n BackupIOError rc: 106\n {}: {}\n BackupFileNotFoundError rc: 107\n {}: {}\n\nOperations\n - cache.begin_transaction\n - cache.download_chunks, appears with ``borg create --no-cache-sync``\n - cache.commit\n - cache.sync\n\n *info* is one string element, the name of the archive currently synced.\n - repository.compact_segments\n - repository.replay_segments\n - repository.check\n - check.verify_data\n - check.rebuild_manifest\n - check.rebuild_refcounts\n - extract\n\n *info* is one string element, the name of the path currently extracted.\n - extract.permissions\n - archive.delete\n - archive.calc_stats\n - prune\n - upgrade.convert_segments\n\nPrompts\n BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK\n For \"Warning: Attempting to access a previously unknown unencrypted repository\"\n BORG_RELOCATED_REPO_ACCESS_IS_OK\n For \"Warning: The repository at location ... was previously located at ...\"\n BORG_CHECK_I_KNOW_WHAT_I_AM_DOING\n For \"This is a potentially dangerous function...\" (check --repair)\n BORG_DELETE_I_KNOW_WHAT_I_AM_DOING\n For \"You requested to DELETE the repository completely *including* all archives it contains:\"\n" }, { "path": "docs/internals/security.rst", "content": ".. include:: ../global.rst.inc\n\n.. somewhat surprisingly the \"bash\" highlighter gives nice results with\n the pseudo-code notation used in the \"Encryption\" section.\n\n.. highlight:: bash\n\n========\nSecurity\n========\n\n.. _borgcrypto:\n\nCryptography in Borg\n====================\n\n.. _attack_model:\n\nAttack model\n------------\n\nThe attack model of Borg is that the environment of the client process\n(e.g. ``borg create``) is trusted and the repository (server) is not. The\nattacker has any and all access to the repository, including interactive\nmanipulation (man-in-the-middle) for remote repositories.\n\nFurthermore, the client environment is assumed to be persistent across\nattacks (practically this means that the security database cannot be\ndeleted between attacks).\n\nUnder these circumstances Borg guarantees that the attacker cannot\n\n1. modify the data of any archive without the client detecting the change\n2. rename or add an archive without the client detecting the change\n3. recover plain-text data\n4. recover definite (heuristics based on access patterns are possible)\n structural information such as the object graph (which archives\n refer to what chunks)\n\nThe attacker can always impose a denial of service by definition (they could\nblock connections to the repository, or delete it partly or entirely).\n\n\n.. _security_structural_auth:\n\nStructural Authentication\n-------------------------\n\nBorg is fundamentally based on an object graph structure (see :ref:`internals`),\nwhere the root objects are the archives.\n\nBorg follows the `Horton principle`_, which states that\nnot only the message must be authenticated, but also its meaning (often\nexpressed through context), because every object used is referenced by a\nparent object through its object ID up to the archive list entry. The object ID in\nBorg is a MAC of the object's plaintext, therefore this ensures that\nan attacker cannot change the context of an object without forging the MAC.\n\nIn other words, the object ID itself only authenticates the plaintext of the\nobject and not its context or meaning. The latter is established by a different\nobject referring to an object ID, thereby assigning a particular meaning to\nan object. For example, an archive item contains a list of object IDs that\nrepresent packed file metadata. On their own, it's not clear that these objects\nwould represent what they do, but by the archive item referring to them\nin a particular part of its own data structure assigns this meaning.\n\nThis results in a directed acyclic graph of authentication from the archive\nlist entry to the data chunks of individual files.\n\nAbove used to be all for borg 1.x and was the reason why it needed the\ntertiary authentication mechanism (TAM) for manifest and archives.\n\nborg 2 now stores the ro_type (\"meaning\") of a repo object's data into that\nobject's metadata (like e.g.: manifest vs. archive vs. user file content data).\nWhen loading data from the repo, borg verifies that the type of object it got\nmatches the type it wanted. borg 2 does not use TAMs any more.\n\nAs both the object's metadata and data are AEAD encrypted and also bound to\nthe object ID (via giving the ID as AAD), there is no way an attacker (without\naccess to the borg key) could change the type of the object or move content\nto a different object ID.\n\nThis effectively 'anchors' each archive to the key, which is controlled by the\nclient, thereby anchoring the DAG starting from the archives list entry,\nmaking it impossible for an attacker to add or modify any part of the\nDAG without Borg being able to detect the tampering.\n\nPlease note that removing an archive by removing an entry from archives/*\nis possible and is done by ``borg delete`` and ``borg prune`` within their\nnormal operation. An attacker could also remove some entries there, but, due to\nencryption, would not know what exactly they are removing. An attacker with\nrepository access could also remove other parts of the repository or the whole\nrepository, so there is not much point in protecting against archive removal.\n\nThe borg 1.x way of having the archives list within the manifest chunk was\nproblematic as it required a read-modify-write operation on the manifest,\nrequiring a lock on the repository. We want to try less locking and more\nparallelism in future.\n\nPassphrase notes\n----------------\n\nNote that when using BORG_PASSPHRASE the attacker cannot swap the *entire*\nrepository against a new repository with e.g. repokey mode and no passphrase,\nbecause Borg will abort access when BORG_PASSPHRASE is incorrect.\n\nHowever, interactively a user might not notice this kind of attack\nimmediately, if she assumes that the reason for the absent passphrase\nprompt is a set BORG_PASSPHRASE. See issue :issue:`2169` for details.\n\n.. _security_encryption:\n\nEncryption\n----------\n\nAEAD modes\n~~~~~~~~~~\n\nModes: --encryption (repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305)\n\nSupported: borg 2.0+\n\nEncryption with these modes is based on AEAD ciphers (authenticated encryption\nwith associated data) and session keys.\n\nDepending on the chosen mode (see :ref:`borg_repo-create`) different AEAD ciphers are used:\n\n- AES-256-OCB - super fast, single-pass algorithm IF you have hw accelerated AES.\n- chacha20-poly1305 - very fast, purely software based AEAD cipher.\n\nThe chunk ID is derived via a MAC over the plaintext (mac key taken from borg key):\n\n- HMAC-SHA256 - super fast IF you have hw accelerated SHA256 (see section \"Encryption\" below).\n- Blake2b - very fast, purely software based algorithm.\n\nFor each borg invocation, a new session id is generated by `os.urandom`_.\n\nFrom that session id, the initial key material (ikm, taken from the borg key)\nand an application and cipher specific salt, borg derives a session key using a\n\"one-step KDF\" based on just sha256.\n\nFor each session key, IVs (nonces) are generated by a counter which increments for\neach encrypted message.\n\nSession::\n\n sessionid = os.urandom(24)\n domain = \"borg-session-key-CIPHERNAME\"\n sessionkey = sha256(crypt_key + sessionid + domain)\n message_iv = 0\n\nEncryption::\n\n id = MAC(id_key, data)\n compressed = compress(data)\n\n header = type-byte || 00h || message_iv || sessionid\n aad = id || header\n message_iv++\n encrypted, auth_tag = AEAD_encrypt(session_key, message_iv, compressed, aad)\n authenticated = header || auth_tag || encrypted\n\nDecryption::\n\n # Given: input *authenticated* data and a *chunk-id* to assert\n type-byte, past_message_iv, past_sessionid, auth_tag, encrypted = SPLIT(authenticated)\n\n ASSERT(type-byte is correct)\n\n domain = \"borg-session-key-CIPHERNAME\"\n past_key = sha256(crypt_key + past_sessionid + domain)\n\n decrypted = AEAD_decrypt(past_key, past_message_iv, authenticated)\n\n decompressed = decompress(decrypted)\n\nNotable:\n\n- More modern and often faster AEAD ciphers instead of self-assembled stuff.\n- Due to the usage of session keys, IVs (nonces) do not need special care here as\n they did for the legacy encryption modes.\n- The id is now also input into the authentication tag computation.\n This strongly associates the id with the written data (== associates the key with\n the value). When later reading the data for some id, authentication will only\n succeed if what we get was really written by us for that id.\n\n\nLegacy modes\n~~~~~~~~~~~~\n\nModes: --encryption (repokey|keyfile)-[blake2]\n\nSupported: borg < 2.0\n\nThese were the AES-CTR based modes in previous borg versions.\n\nborg 2.0 does not support creating new repos using these modes,\nbut ``borg transfer`` can still read such existing repos.\n\n\n.. _key_encryption:\n\nOffline key security\n--------------------\n\nBorg cannot secure the key material while it is running, because the keys\nare needed in plain to decrypt/encrypt repository objects.\n\nFor offline storage of the encryption keys they are encrypted with a\nuser-chosen passphrase.\n\nA 256 bit key encryption key (KEK) is derived from the passphrase\nusing argon2_ with a random 256 bit salt. The KEK is then used\nto Encrypt-*then*-MAC a packed representation of the keys using the\nchacha20-poly1305 AEAD cipher and a constant IV == 0.\nThe ciphertext is then converted to base64.\n\nThis base64 blob (commonly referred to as *keyblob*) is then stored in\nthe key file or in the repository config (keyfile and repokey modes\nrespectively).\n\nThe use of a constant IV is secure because an identical passphrase will\nresult in a different derived KEK for every key encryption due to the salt.\n\n\n.. seealso::\n\n Refer to the :ref:`key_files` section for details on the format.\n\n\nImplementations used\n--------------------\n\nWe do not implement cryptographic primitives ourselves, but rely\non widely used libraries providing them:\n\n- AES-OCB and CHACHA20-POLY1305 from OpenSSL 1.1 are used,\n which is also linked into the static binaries we provide.\n We think this is not an additional risk, since we don't ever\n use OpenSSL's networking, TLS or X.509 code, but only their\n primitives implemented in libcrypto.\n- SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used.\n- HMAC and a constant-time comparison from Python's hmac_ standard library module are used.\n- argon2 is used via argon2-cffi.\n\n.. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle\n.. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack\n.. _hashlib: https://docs.python.org/3/library/hashlib.html\n.. _hmac: https://docs.python.org/3/library/hmac.html\n.. _os.urandom: https://docs.python.org/3/library/os.html#os.urandom\n\nRemote RPC protocol security\n============================\n\n.. note:: This section could be further expanded / detailed.\n\nThe RPC protocol is fundamentally based on msgpack'd messages exchanged\nover an encrypted SSH channel (the system's SSH client is used for this\nby piping data from/to it).\n\nThis means that the authorization and transport security properties\nare inherited from SSH and the configuration of the SSH client and the\nSSH server -- Borg RPC does not contain *any* networking\ncode. Networking is done by the SSH client running in a separate\nprocess, Borg only communicates over the standard pipes (stdout,\nstderr and stdin) with this process. This also means that Borg doesn't\nhave to use a SSH client directly (or SSH at all). For example,\n``sudo`` or ``qrexec`` could be used as an intermediary.\n\nBy using the system's SSH client and not implementing a\n(cryptographic) network protocol Borg sidesteps many security issues\nthat would normally impact distributing statically linked / standalone\nbinaries.\n\nThe remainder of this section will focus on the security of the RPC\nprotocol within Borg.\n\nThe assumed worst-case a server can inflict to a client is a\ndenial of repository service.\n\nThe situation where a server can create a general DoS on the client\nshould be avoided, but might be possible by e.g. forcing the client to\nallocate large amounts of memory to decode large messages (or messages\nthat merely indicate a large amount of data follows). The RPC protocol\ncode uses a limited msgpack Unpacker to prohibit this.\n\nWe believe that other kinds of attacks, especially critical vulnerabilities\nlike remote code execution are inhibited by the design of the protocol:\n\n1. The server cannot send requests to the client on its own accord,\n it only can send responses. This avoids \"unexpected inversion of control\"\n issues.\n2. msgpack serialization does not allow embedding or referencing code that\n is automatically executed. Incoming messages are unpacked by the msgpack\n unpacker into native Python data structures (like tuples and dictionaries),\n which are then passed to the rest of the program.\n\n Additional verification of the correct form of the responses could be implemented.\n3. Remote errors are presented in two forms:\n\n 1. A simple plain-text *stderr* channel. A prefix string indicates the kind of message\n (e.g. WARNING, INFO, ERROR), which is used to suppress it according to the\n log level selected in the client.\n\n A server can send arbitrary log messages, which may confuse a user. However,\n log messages are only processed when server requests are in progress, therefore\n the server cannot interfere / confuse with security critical dialogue like\n the password prompt.\n 2. Server-side exceptions passed over the main data channel. These follow the\n general pattern of server-sent responses and are sent instead of response data\n for a request.\n\nThe msgpack implementation used (msgpack-python) has a good security track record,\na large test suite and no issues found by fuzzing. It is based on the msgpack-c implementation,\nsharing the unpacking engine and some support code. msgpack-c has a good track record as well.\nSome issues [#]_ in the past were located in code not included in msgpack-python.\nBorg does not use msgpack-c.\n\n.. [#] - `MessagePack fuzzing `_\n - `Fixed integer overflow and EXT size problem `_\n - `Fixed array and map size overflow `_\n\nUsing OpenSSL\n=============\n\nBorg uses the OpenSSL library for most cryptography (see `Implementations used`_ above).\nOpenSSL is bundled with static releases, thus the bundled copy is not updated with system\nupdates.\n\nOpenSSL is a large and complex piece of software and has had its share of vulnerabilities,\nhowever, it is important to note that Borg links against ``libcrypto`` **not** ``libssl``.\nlibcrypto is the low-level cryptography part of OpenSSL,\nwhile libssl implements TLS and related protocols.\n\nThe latter is not used by Borg (cf. `Remote RPC protocol security`_, Borg itself does not implement\nany network access) and historically contained most vulnerabilities, especially critical ones.\nThe static binaries released by the project contain neither libssl nor the Python ssl/_ssl modules.\n\nCompression and Encryption\n==========================\n\nCombining encryption with compression can be insecure in some contexts (e.g. online protocols).\n\nThere was some discussion about this in :issue:`1040` and for Borg some developers\nconcluded this is no problem at all, some concluded this is hard and extremely slow to exploit\nand thus no problem in practice.\n\nNo matter what, there is always the option not to use compression if you are worried about this.\n\n\nFingerprinting\n==============\n\nStored chunk sizes\n------------------\n\nA borg repository does not hide the size of the chunks it stores (size\ninformation is needed to operate the repository).\n\nThe chunks stored in the repo are the (compressed, encrypted and authenticated)\noutput of the chunker. The sizes of these stored chunks are influenced by the\ncompression, encryption and authentication.\n\nbuzhash and buzhash64 chunker\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe buzhash chunkers chunk according to the input data, the chunker's\nparameters and secret key material (which all influence the chunk boundary\npositions).\n\nSecret key material:\n\n- \"buzhash\": chunker seed (32bits), used for XORing the hardcoded buzhash table\n- \"buzhash64\": bh64_key (256bits) is derived from ID key, used to cryptographically\n generate the table.\n\nSmall files below some specific threshold (default: 512 KiB) result in only one\nchunk (identical content / size as the original file), bigger files result in\nmultiple chunks.\n\nfixed chunker\n~~~~~~~~~~~~~\n\nThis chunker yields fixed sized chunks, with optional support of a differently\nsized header chunk. The last chunk is not required to have the full block size\nand is determined by the input file size.\n\nWithin our attack model, an attacker possessing a specific set of files which\nhe assumes that the victim also possesses (and backups into the repository)\ncould try a brute force fingerprinting attack based on the chunk sizes in the\nrepository to prove his assumption.\n\nTo make this more difficult, borg has an ``obfuscate`` pseudo compressor, that\nwill take the output of the normal compression step and tries to obfuscate\nthe size of that output. Of course, it can only **add** to the size, not reduce\nit. Thus, the optional usage of this mechanism comes at a cost: it will make\nyour repository larger (ranging from a few percent larger [cheap] to ridiculously\nlarger [expensive], depending on the algorithm/params you wisely choose).\n\nThe output of the compressed-size obfuscation step will then be encrypted and\nauthenticated, as usual. Of course, using that obfuscation would not make any\nsense without encryption. Thus, the additional data added by the obfuscator\nare just 0x00 bytes, which is good enough because after encryption it will\nlook like random anyway.\n\nTo summarize, this is making size-based fingerprinting difficult:\n\n- user-selectable chunker algorithm (and parametrization)\n- for the buzhash chunker: secret, random per-repo chunker seed\n- user-selectable compression algorithm (and level)\n- optional ``obfuscate`` pseudo compressor with different choices\n of algorithm and parameters\n\nSecret key usage against fingerprinting\n---------------------------------------\n\nBorg uses the borg key also for chunking and chunk ID generation to protect against fingerprinting.\nAs usual for borg's attack model, the attacker is assumed to have access to a borg repository.\n\nThe borg key includes a secret random chunk_seed which (together with the chunking algorithm)\ndetermines the cutting places and thereby the length of the chunks cut. Because the attacker trying\na chunk length fingerprinting attack would use a different chunker secret than the borg setup being\nattacked, they would not be able to determine the set of chunk lengths for a known set of files.\n\nThe borg key also includes a secret random id_key. The chunk ID generation is not just using a simple\ncryptographic hash like sha256 (because that would be insecure as an attacker could see the hashes of\nsmall files that result only in 1 chunk in the repository). Instead, borg uses keyed hash (a MAC,\ne.g. HMAC-SHA256) to compute the chunk ID from the content and the secret id_key. Thus, an attacker\ncan't compute the same chunk IDs for a known set of small files to determine whether these are stored\nin the attacked repository.\n\nStored chunk proximity\n----------------------\n\nBorg does not try to obfuscate order / proximity of files it discovers by\nrecursing through the filesystem. For performance reasons, we sort directory\ncontents in file inode order (not in file name alphabetical order), so order\nfingerprinting is not useful for an attacker.\n\nBut, when new files are close to each other (when looking at recursion /\nscanning order), the resulting chunks will be also stored close to each other\nin the resulting repository segment file(s).\n\nThis might leak additional information for the chunk size fingerprinting\nattack (see above).\n" }, { "path": "docs/internals.rst", "content": ".. include:: global.rst.inc\n.. _internals:\n\nInternals\n=========\n\nThe internals chapter describes and analyzes most of the inner workings\nof Borg.\n\nBorg uses a low-level, key-value store, the :ref:`repository`, and\nimplements a more complex data structure on top of it, which is made\nup of the :ref:`manifest `, :ref:`archives `,\n:ref:`items ` and data :ref:`chunks`.\n\nEach repository can hold multiple :ref:`archives `, which\nrepresent individual backups that contain a full archive of the files\nspecified when the backup was performed.\n\nDeduplication is performed globally across all data in the repository\n(multiple backups and even multiple hosts), both on data and file\nmetadata, using :ref:`chunks` created by the chunker using the\nBuzhash_ algorithm (\"buzhash\" and \"buzhash64\" chunker) or a simpler\nfixed block size algorithm (\"fixed\" chunker).\n\nTo perform the repository-wide deduplication, a hash of each\nchunk is checked against the :ref:`chunks cache `, which is a\nhash table of all chunks that already exist.\n\n.. figure:: internals/structure.png\n :figwidth: 100%\n :width: 100%\n\n Layers in Borg. At the very top, commands are implemented, using\n a data access layer provided by the Archive and Item classes.\n The \"key\" object provides both compression and authenticated\n encryption used by the data access layer. The \"key\" object represents\n the sole trust boundary in Borg.\n The lowest layer is the repository, either accessed directly\n (Repository) or remotely (RemoteRepository).\n\n.. toctree::\n :caption: Internals contents\n\n internals/security\n internals/data-structures\n internals/frontends\n" }, { "path": "docs/introduction.rst", "content": "Introduction\n============\n\n.. This shim is here to fix the structure in the PDF\n rendering. Without this stub, the elements in the toctree of\n index.rst show up a level below the README file included.\n\n.. include:: ../README.rst\n" }, { "path": "docs/man/borg-analyze.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-analyze\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-analyze \\- Analyzes archives.\n.SH SYNOPSIS\n.sp\nborg [common options] analyze [options]\n.SH DESCRIPTION\n.sp\nAnalyze archives to find \\(dqhot spots\\(dq.\n.sp\n\\fBborg analyze\\fP relies on the usual archive matching options to select the\narchives that should be considered for analysis (e.g. \\fB\\-a series_name\\fP).\nThen it iterates over all matching archives, over all contained files, and\ncollects information about chunks stored in all directories it encounters.\n.sp\nIt considers chunk IDs and their plaintext sizes (we do not have the compressed\nsize in the repository easily available) and adds up the sizes of added and removed\nchunks per direct parent directory, and outputs a list of \\(dqdirectory: size\\(dq.\n.sp\nYou can use that list to find directories with a lot of \\(dqactivity\\(dq — maybe\nsome of these are temporary or cache directories you forgot to exclude.\n.sp\nTo avoid including these unwanted directories in your backups, you can carefully\nexclude them in \\fBborg create\\fP (for future backups) or use \\fBborg recreate\\fP\nto recreate existing archives without them.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS Archive filters\n.INDENT 0.0\n.TP\n.BI \\-a \\ PATTERN\\fR,\\fB \\ \\-\\-match\\-archives \\ PATTERN\nonly consider archives matching all patterns. See \\(dqborg help match\\-archives\\(dq.\n.TP\n.BI \\-\\-sort\\-by \\ KEYS\nComma\\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp\n.TP\n.BI \\-\\-first \\ N\nconsider the first N archives after other filters are applied\n.TP\n.BI \\-\\-last \\ N\nconsider the last N archives after other filters are applied\n.TP\n.BI \\-\\-oldest \\ TIMESPAN\nconsider archives between the oldest archive\\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newest \\ TIMESPAN\nconsider archives between the newest archive\\(aqs timestamp and (newest \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-older \\ TIMESPAN\nconsider archives older than (now \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newer \\ TIMESPAN\nconsider archives newer than (now \\- TIMESPAN), e.g., 7d or 12m.\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-benchmark-cpu.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-benchmark-cpu\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-benchmark-cpu \\- Benchmark CPU-bound operations.\n.SH SYNOPSIS\n.sp\nborg [common options] benchmark cpu [options]\n.SH DESCRIPTION\n.sp\nThis command benchmarks miscellaneous CPU\\-bound Borg operations.\n.sp\nIt creates input data in memory, runs the operation and then displays throughput.\nTo reduce outside influence on the timings, please make sure to run this with:\n.INDENT 0.0\n.IP \\(bu 2\nan otherwise as idle as possible machine\n.IP \\(bu 2\nenough free memory so there will be no slow down due to paging activity\n.UNINDENT\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-json\nformat output as JSON\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-benchmark-crud.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-benchmark-crud\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-benchmark-crud \\- Benchmark Create, Read, Update, Delete for archives.\n.SH SYNOPSIS\n.sp\nborg [common options] benchmark crud [options] PATH\n.SH DESCRIPTION\n.sp\nThis command benchmarks borg CRUD (create, read, update, delete) operations.\n.sp\nIt creates input data below the given PATH and backs up this data into the given REPO.\nThe REPO must already exist (it could be a fresh empty repo or an existing repo, the\ncommand will create / read / update / delete some archives named borg\\-benchmark\\-crud* there.\n.sp\nMake sure you have free space there; you will need about 1 GB each (+ overhead).\n.sp\nIf your repository is encrypted and borg needs a passphrase to unlock the key, use:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nBORG_PASSPHRASE=mysecret borg benchmark crud REPO PATH\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nMeasurements are done with different input file sizes and counts.\nThe file contents are very artificial (either all zero or all random),\nthus the measurement results do not necessarily reflect performance with real data.\nAlso, due to the kind of content used, no compression is used in these benchmarks.\n.INDENT 0.0\n.TP\n.B C\\- == borg create (1st archive creation, no compression, do not use files cache)\nC\\-Z\\- == all\\-zero files. full dedup, this is primarily measuring reader/chunker/hasher.\nC\\-R\\- == random files. no dedup, measuring throughput through all processing stages.\n.TP\n.B R\\- == borg extract (extract archive, dry\\-run, do everything, but do not write files to disk)\nR\\-Z\\- == all zero files. Measuring heavily duplicated files.\nR\\-R\\- == random files. No duplication here, measuring throughput through all processing\nstages, except writing to disk.\n.TP\n.B U\\- == borg create (2nd archive creation of unchanged input files, measure files cache speed)\nThe throughput value is kind of virtual here, it does not actually read the file.\nU\\-Z\\- == needs to check the 2 all\\-zero chunks\\(aq existence in the repo.\nU\\-R\\- == needs to check existence of a lot of different chunks in the repo.\n.TP\n.B D\\- == borg delete archive (delete last remaining archive, measure deletion + compaction)\nD\\-Z\\- == few chunks to delete / few segments to compact/remove.\nD\\-R\\- == many chunks to delete / many segments to compact/remove.\n.UNINDENT\n.sp\nPlease note that there might be quite some variance in these measurements.\nTry multiple measurements and having a otherwise idle machine (and network, if you use it).\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B PATH\npath where to create benchmark input data\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-json\\-lines\nFormat output as JSON Lines.\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-benchmark.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-benchmark\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-benchmark \\- benchmark command\n.SH SYNOPSIS\n.nf\nborg [common options] benchmark crud ...\nborg [common options] benchmark cpu ...\n.fi\n.sp\n.SH DESCRIPTION\n.sp\nThese commands do various benchmarks.\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-benchmark\\-crud(1)\\fP, \\fIborg\\-benchmark\\-cpu(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-break-lock.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-break-lock\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-break-lock \\- Breaks the repository lock (for example, if it was left by a dead Borg process).\n.SH SYNOPSIS\n.sp\nborg [common options] break\\-lock [options]\n.SH DESCRIPTION\n.sp\nThis command breaks the repository and cache locks.\nUse with care and only when no Borg process (on any machine) is\ntrying to access the cache or the repository.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-check.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-check\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-check \\- Checks repository consistency.\n.SH SYNOPSIS\n.sp\nborg [common options] check [options]\n.SH DESCRIPTION\n.sp\nThe check command verifies the consistency of a repository and its archives.\nIt consists of two major steps:\n.INDENT 0.0\n.IP 1. 3\nChecking the consistency of the repository itself. This includes checking\nthe file magic headers, and both the metadata and data of all objects in\nthe repository. The read data is checked by size and hash. Bit rot and other\ntypes of accidental damage can be detected this way. Running the repository\ncheck can be split into multiple partial checks using \\fB\\-\\-max\\-duration\\fP\\&.\nWhen checking an \\% remote repository, please note that the checks run on\nthe server and do not cause significant network traffic.\n.IP 2. 3\nChecking consistency and correctness of the archive metadata and optionally\narchive data (requires \\fB\\-\\-verify\\-data\\fP). This includes ensuring that the\nrepository manifest exists, the archive metadata chunk is present, and that\nall chunks referencing files (items) in the archive exist. This requires\nreading archive and file metadata, but not data. To scan for archives whose\nentries were lost from the archive directory, pass \\fB\\-\\-find\\-lost\\-archives\\fP\\&.\nIt requires reading all data and is hence very time\\-consuming.\nTo additionally cryptographically verify the file (content) data integrity,\npass \\fB\\-\\-verify\\-data\\fP, which is even more time\\-consuming.\n.sp\nWhen checking archives of a remote repository, archive checks run on the client\nmachine because they require decrypting data and therefore the encryption key.\n.UNINDENT\n.sp\nBoth steps can also be run independently. Pass \\fB\\-\\-repository\\-only\\fP to run the\nrepository checks only, or pass \\fB\\-\\-archives\\-only\\fP to run the archive checks\nonly.\n.sp\nThe \\fB\\-\\-max\\-duration\\fP option can be used to split a long\\-running repository\ncheck into multiple partial checks. After the given number of seconds, the check\nis interrupted. The next partial check will continue where the previous one\nstopped, until the full repository has been checked. Assuming a complete check\nwould take 7 hours, then running a daily check with \\fB\\-\\-max\\-duration=3600\\fP\n(1 hour) would result in one full repository check per week. Doing a full\nrepository check aborts any previous partial check; the next partial check will\nrestart from the beginning. With partial repository checks you can run neither\narchive checks, nor enable repair mode. Consequently, if you want to use\n\\fB\\-\\-max\\-duration\\fP you must also pass \\fB\\-\\-repository\\-only\\fP, and must not pass\n\\fB\\-\\-archives\\-only\\fP, nor \\fB\\-\\-repair\\fP\\&.\n.sp\n\\fBWarning:\\fP Please note that partial repository checks (i.e., running with\n\\fB\\-\\-max\\-duration\\fP) can only perform non\\-cryptographic checksum checks on the\nrepository files. Enabling partial repository checks excludes archive checks\nfor the same reason. Therefore, partial checks may be useful only with very large\nrepositories where a full check would take too long.\n.sp\nThe \\fB\\-\\-verify\\-data\\fP option will perform a full integrity verification (as\nopposed to checking just the xxh64) of data, which means reading the\ndata from the repository, decrypting and decompressing it. It is a complete\ncryptographic verification and hence very time\\-consuming, but will detect any\naccidental and malicious corruption. Tamper\\-resistance is only guaranteed for\nencrypted repositories against attackers without access to the keys. You cannot\nuse \\fB\\-\\-verify\\-data\\fP with \\fB\\-\\-repository\\-only\\fP\\&.\n.sp\nThe \\fB\\-\\-find\\-lost\\-archives\\fP option will also scan the whole repository, but\ntells Borg to search for lost archive metadata. If Borg encounters any archive\nmetadata that does not match an archive directory entry (including\nsoft\\-deleted archives), it means that an entry was lost.\nUnless \\fBborg compact\\fP is called, these archives can be fully restored with\n\\fB\\-\\-repair\\fP\\&. Please note that \\fB\\-\\-find\\-lost\\-archives\\fP must read a lot of\ndata from the repository and is thus very time\\-consuming. You cannot use\n\\fB\\-\\-find\\-lost\\-archives\\fP with \\fB\\-\\-repository\\-only\\fP\\&.\n.SS About repair mode\n.sp\nThe check command is a read\\-only task by default. If any corruption is found,\nBorg will report the issue and proceed with checking. To actually repair the\nissues found, pass \\fB\\-\\-repair\\fP\\&.\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\n\\fB\\-\\-repair\\fP is a \\fBPOTENTIALLY DANGEROUS FEATURE\\fP and might lead to data\nloss! This does not just include data that was previously lost anyway, but\nmight include more data for kinds of corruption it is not capable of\ndealing with. \\fBBE VERY CAREFUL!\\fP\n.UNINDENT\n.UNINDENT\n.sp\nPursuant to the previous warning it is also highly recommended to test the\nreliability of the hardware running Borg with stress testing software. This\nespecially includes storage and memory testers. Unreliable hardware might lead\nto additional data loss.\n.sp\nIt is highly recommended to create a backup of your repository before running\nin repair mode (i.e. running it with \\fB\\-\\-repair\\fP).\n.sp\nRepair mode will attempt to fix any corruptions found. Fixing corruptions does\nnot mean recovering lost data: Borg cannot magically restore data lost due to\ne.g. a hardware failure. Repairing a repository means sacrificing some data\nfor the sake of the repository as a whole and the remaining data. Hence it is,\nby definition, a potentially lossy task.\n.sp\nIn practice, repair mode hooks into both the repository and archive checks:\n.INDENT 0.0\n.IP 1. 3\nWhen checking the repository\\(aqs consistency, repair mode removes corrupted\nobjects from the repository after it did a 2nd try to read them correctly.\n.IP 2. 3\nWhen checking the consistency and correctness of archives, repair mode might\nremove whole archives from the manifest if their archive metadata chunk is\ncorrupt or lost. Borg will also report files that reference missing chunks.\n.UNINDENT\n.sp\nIf \\fB\\-\\-repair \\-\\-find\\-lost\\-archives\\fP is given, previously lost entries will\nbe recreated in the archive directory. This is only possible before\n\\fBborg compact\\fP would remove the archives\\(aq data completely.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-repository\\-only\nonly perform repository checks\n.TP\n.B \\-\\-archives\\-only\nonly perform archive checks\n.TP\n.B \\-\\-verify\\-data\nperform cryptographic archive data integrity verification (conflicts with \\fB\\-\\-repository\\-only\\fP)\n.TP\n.B \\-\\-repair\nattempt to repair any inconsistencies found\n.TP\n.B \\-\\-find\\-lost\\-archives\nattempt to find lost archives\n.TP\n.BI \\-\\-max\\-duration \\ SECONDS\nperform only a partial repository check for at most SECONDS seconds (default: unlimited)\n.UNINDENT\n.SS Archive filters\n.INDENT 0.0\n.TP\n.BI \\-a \\ PATTERN\\fR,\\fB \\ \\-\\-match\\-archives \\ PATTERN\nonly consider archives matching all patterns. See \\(dqborg help match\\-archives\\(dq.\n.TP\n.BI \\-\\-sort\\-by \\ KEYS\nComma\\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp\n.TP\n.BI \\-\\-first \\ N\nconsider the first N archives after other filters are applied\n.TP\n.BI \\-\\-last \\ N\nconsider the last N archives after other filters are applied\n.TP\n.BI \\-\\-oldest \\ TIMESPAN\nconsider archives between the oldest archive\\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newest \\ TIMESPAN\nconsider archives between the newest archive\\(aqs timestamp and (newest \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-older \\ TIMESPAN\nconsider archives older than (now \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newer \\ TIMESPAN\nconsider archives newer than (now \\- TIMESPAN), e.g., 7d or 12m.\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-common.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-common\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-common \\- Common options of Borg commands\n.SH SYNOPSIS\n.INDENT 0.0\n.TP\n.B \\-h\\fP,\\fB \\-\\-help\nshow this help message and exit\n.TP\n.B \\-\\-critical\nwork on log level CRITICAL\n.TP\n.B \\-\\-error\nwork on log level ERROR\n.TP\n.B \\-\\-warning\nwork on log level WARNING (default)\n.TP\n.B \\-\\-info\\fP,\\fB \\-v\\fP,\\fB \\-\\-verbose\nwork on log level INFO\n.TP\n.B \\-\\-debug\nenable debug output, work on log level DEBUG\n.TP\n.BI \\-\\-debug\\-topic \\ TOPIC\nenable TOPIC debugging (can be specified multiple times). The logger path is borg.debug. if TOPIC is not fully qualified.\n.TP\n.B \\-p\\fP,\\fB \\-\\-progress\nshow progress information\n.TP\n.B \\-\\-iec\nformat using IEC units (1KiB = 1024B)\n.TP\n.B \\-\\-log\\-json\nOutput one JSON object per log line instead of formatted text.\n.TP\n.BI \\-\\-lock\\-wait \\ SECONDS\nwait at most SECONDS for acquiring a repository/cache lock (default: 10).\n.TP\n.B \\-\\-show\\-version\nshow/log the borg version\n.TP\n.B \\-\\-show\\-rc\nshow/log the return code (rc)\n.TP\n.BI \\-\\-umask \\ M\nset umask to M (local only, default: 0077)\n.TP\n.BI \\-\\-remote\\-path \\ PATH\nuse PATH as borg executable on the remote (default: \\(dqborg\\(dq)\n.TP\n.BI \\-\\-upload\\-ratelimit \\ RATE\nset network upload rate limit in kiByte/s (default: 0=unlimited)\n.TP\n.BI \\-\\-upload\\-buffer \\ UPLOAD_BUFFER\nset network upload buffer size in MiB. (default: 0=no buffer)\n.TP\n.BI \\-\\-debug\\-profile \\ FILE\nWrite execution profile in Borg format into FILE. For local use a Python\\-compatible file can be generated by suffixing FILE with \\(dq.pyprof\\(dq.\n.TP\n.BI \\-\\-rsh \\ RSH\nUse this command to connect to the \\(aqborg serve\\(aq process (default: \\(aqssh\\(aq)\n.TP\n.BI \\-\\-socket \\ PATH\nUse UNIX DOMAIN (IPC) socket at PATH for client/server communication with socket: protocol.\n.TP\n.BI \\-r \\ REPO\\fR,\\fB \\ \\-\\-repo \\ REPO\nrepository to use\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-compact.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-compact\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-compact \\- Collects garbage in the repository.\n.SH SYNOPSIS\n.sp\nborg [common options] compact [options]\n.SH DESCRIPTION\n.sp\nFree repository space by deleting unused chunks.\n.sp\n\\fBborg compact\\fP analyzes all existing archives to determine which repository\nobjects are actually used (referenced). It then deletes all unused objects\nfrom the repository to free space.\n.sp\nUnused objects may result from:\n.INDENT 0.0\n.IP \\(bu 2\nuse of \\fBborg delete\\fP or \\fBborg prune\\fP\n.IP \\(bu 2\ninterrupted backups (consider retrying the backup before running compact)\n.IP \\(bu 2\nbackups of source files that encountered an I/O error mid\\-transfer and were skipped\n.IP \\(bu 2\ncorruption of the repository (e.g., the archives directory lost entries; see notes below)\n.UNINDENT\n.sp\nYou usually do not want to run \\fBborg compact\\fP after every write operation, but\neither regularly (e.g., once a month, possibly together with \\fBborg check\\fP) or\nwhen disk space needs to be freed.\n.sp\n\\fBImportant:\\fP\n.sp\nAfter compacting, it is no longer possible to use \\fBborg undelete\\fP to recover\npreviously soft\\-deleted archives.\n.sp\n\\fBborg compact\\fP might also delete data from archives that were \\(dqlost\\(dq due to\narchives directory corruption. Such archives could potentially be restored with\n\\fBborg check \\-\\-find\\-lost\\-archives [\\-\\-repair]\\fP, which is slow. You therefore\nmight not want to do that unless there are signs of lost archives (e.g., when\nseeing fatal errors when creating backups or when archives are missing in\n\\fBborg repo\\-list\\fP).\n.sp\nWhen using the \\fB\\-\\-stats\\fP option, borg will internally list all repository\nobjects to determine their existence and stored size. It will build a fresh\nchunks index from that information and cache it in the repository. For some\ntypes of repositories, this might be very slow. It will tell you the sum of\nstored object sizes, before and after compaction.\n.sp\nWithout \\fB\\-\\-stats\\fP, borg will rely on the cached chunks index to determine\nexisting object IDs (but there is no stored size information in the index,\nthus it cannot compute before/after compaction size statistics).\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS options\n.INDENT 0.0\n.TP\n.B \\-n\\fP,\\fB \\-\\-dry\\-run\ndo not change the repository\n.TP\n.B \\-s\\fP,\\fB \\-\\-stats\nprint statistics (might be much slower)\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Compact segments and free repository disk space\n$ borg compact\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-completion.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-completion\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-completion \\- Output shell completion script for the given shell.\n.SH SYNOPSIS\n.sp\nborg [common options] completion [options] SHELL\n.SH DESCRIPTION\n.sp\nThis command prints a shell completion script for the given shell.\n.sp\nPlease note that for some dynamic completions (like archive IDs), the shell\ncompletion script will call borg to query the repository. This will work best\nif that call can be made without prompting for user input, so you may want to\nset BORG_REPO and BORG_PASSPHRASE environment variables.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B SHELL\nshell to generate completion for (one of: %(choices)s)\n.UNINDENT\n.SH EXAMPLES\n.sp\nTo activate completion in your current shell session, evaluate the output\nof this command. To enable it persistently, add the corresponding line to\nyour shell\\(aqs startup file.\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Bash (in ~/.bashrc)\neval \\(dq$(borg completion bash)\\(dq\n\n# Zsh (in ~/.zshrc)\neval \\(dq$(borg completion zsh)\\(dq\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-compression.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-compression\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-compression \\- Details regarding compression\n.SH DESCRIPTION\n.sp\nIt is no problem to mix different compression methods in one repository,\ndeduplication is done on the source data chunks (not on the compressed\nor encrypted data).\n.sp\nIf some specific chunk was once compressed and stored into the repository, creating\nanother backup that also uses this chunk will not change the stored chunk.\nSo if you use different compression specs for the backups, whichever stores a\nchunk first determines its compression. See also \\fBborg recreate\\fP\\&.\n.sp\nCompression is lz4 by default. If you want something else, you have to specify what you want.\n.sp\nValid compression specifiers are:\n.INDENT 0.0\n.TP\n.B none\nDo not compress.\n.TP\n.B lz4\nUse lz4 compression. Very high speed, very low compression. (default)\n.TP\n.B zstd[,L]\nUse zstd (\\(dqzstandard\\(dq) compression, a modern wide\\-range algorithm.\nIf you do not explicitly give the compression level L (ranging from 1\nto 22), it will use level 3.\n.TP\n.B zlib[,L]\nUse zlib (\\(dqgz\\(dq) compression. Medium speed, medium compression.\nIf you do not explicitly give the compression level L (ranging from 0\nto 9), it will use level 6.\nGiving level 0 (means \\(dqno compression\\(dq, but still has zlib protocol\noverhead) is usually pointless, you better use \\(dqnone\\(dq compression.\n.TP\n.B lzma[,L]\nUse lzma (\\(dqxz\\(dq) compression. Low speed, high compression.\nIf you do not explicitly give the compression level L (ranging from 0\nto 9), it will use level 6.\nGiving levels above 6 is pointless and counterproductive because it does\nnot compress better due to the buffer size used by borg \\- but it wastes\nlots of CPU cycles and RAM.\n.TP\n.B auto,C[,L]\nUse a built\\-in heuristic to decide per chunk whether to compress or not.\nThe heuristic tries with lz4 whether the data is compressible.\nFor incompressible data, it will not use compression (uses \\(dqnone\\(dq).\nFor compressible data, it uses the given C[,L] compression \\- with C[,L]\nbeing any valid compression specifier. This can be helpful for media files\nwhich often cannot be compressed much more.\n.TP\n.B obfuscate,SPEC,C[,L]\nUse compressed\\-size obfuscation to make fingerprinting attacks based on\nthe observable stored chunk size more difficult. Note:\n.INDENT 7.0\n.IP \\(bu 2\nYou must combine this with encryption, or it won\\(aqt make any sense.\n.IP \\(bu 2\nYour repo size will be bigger, of course.\n.IP \\(bu 2\nA chunk is limited by the constant \\fBMAX_DATA_SIZE\\fP (cur. ~20MiB).\n.UNINDENT\n.sp\nThe SPEC value determines how the size obfuscation works:\n.sp\n\\fIRelative random reciprocal size variation\\fP (multiplicative)\n.sp\nSize will increase by a factor, relative to the compressed data size.\nSmaller factors are used often, larger factors rarely.\n.sp\nAvailable factors:\n.INDENT 7.0\n.INDENT 3.5\n.sp\n.EX\n1: 0.01 .. 100\n2: 0.1 .. 1,000\n3: 1 .. 10,000\n4: 10 .. 100,000\n5: 100 .. 1,000,000\n6: 1,000 .. 10,000,000\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nExample probabilities for SPEC \\fB1\\fP:\n.INDENT 7.0\n.INDENT 3.5\n.sp\n.EX\n90 % 0.01 .. 0.1\n 9 % 0.1 .. 1\n 0.9 % 1 .. 10\n 0.09% 10 .. 100\n.EE\n.UNINDENT\n.UNINDENT\n.sp\n\\fIRandomly sized padding up to the given size\\fP (additive)\n.INDENT 7.0\n.INDENT 3.5\n.sp\n.EX\n110: 1kiB (2 ^ (SPEC \\- 100))\n\\&...\n120: 1MiB\n\\&...\n123: 8MiB (max.)\n.EE\n.UNINDENT\n.UNINDENT\n.sp\n\\fIPadmé padding\\fP (deterministic)\n.INDENT 7.0\n.INDENT 3.5\n.sp\n.EX\n250: pads to sums of powers of 2, max 12% overhead\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nUses the Padmé algorithm to deterministically pad the compressed size to a sum of\npowers of 2, limiting overhead to 12%. See \\% for details.\n.UNINDENT\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nborg create \\-\\-compression lz4 \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression zstd \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression zstd,10 \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression zlib \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression zlib,1 \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression auto,lzma,6 \\-\\-repo REPO ARCHIVE data\nborg create \\-\\-compression auto,lzma ...\nborg create \\-\\-compression obfuscate,110,none ...\nborg create \\-\\-compression obfuscate,3,auto,zstd,10 ...\nborg create \\-\\-compression obfuscate,2,zstd,6 ...\nborg create \\-\\-compression obfuscate,250,zstd,3 ...\n.EE\n.UNINDENT\n.UNINDENT\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-config.1", "content": ".\\\" Man page generated from reStructuredText.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"BORG-CONFIG\" 1 \"2024-07-19\" \"\" \"borg backup tool\"\n.SH NAME\nborg-config \\- get, set, and delete values in a repository or cache config file\n.SH SYNOPSIS\n.sp\nborg [common options] config [options] [NAME] [VALUE]\n.SH DESCRIPTION\n.sp\nThis command gets and sets options in a local repository or cache config file.\nFor security reasons, this command only works on local repositories.\n.sp\nTo delete a config value entirely, use \\fB\\-\\-delete\\fP\\&. To list the values\nof the configuration file or the default values, use \\fB\\-\\-list\\fP\\&. To get an existing\nkey, pass only the key name. To set a key, pass both the key name and\nthe new value. Keys can be specified in the format \\(dqsection.name\\(dq or\nsimply \\(dqname\\(dq; the section will default to \\(dqrepository\\(dq and \\(dqcache\\(dq for\nthe repo and cache configs, respectively.\n.sp\nBy default, borg config manipulates the repository config file. Using \\fB\\-\\-cache\\fP\nedits the repository cache\\(aqs config file instead.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nname of config key\n.TP\n.B VALUE\nnew value for key\n.UNINDENT\n.SS optional arguments\n.INDENT 0.0\n.TP\n.B \\-c\\fP,\\fB \\-\\-cache\nget and set values from the repo cache\n.TP\n.B \\-d\\fP,\\fB \\-\\-delete\ndelete the key from the config file\n.TP\n.B \\-l\\fP,\\fB \\-\\-list\nlist the configuration of the repo\n.UNINDENT\n.SH EXAMPLES\n.sp\n\\fBNOTE:\\fP\n.INDENT 0.0\n.INDENT 3.5\nThe repository & cache config files are some of the only directly manipulable\nparts of a repository that aren\\(aqt versioned or backed up, so be careful when\nmaking changes!\n.UNINDENT\n.UNINDENT\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.nf\n.ft C\n# find cache directory\n$ cd ~/.cache/borg/$(borg config id)\n\n# reserve some space\n$ borg config additional_free_space 2G\n\n# make a repo append\\-only\n$ borg config append_only 1\n.ft P\n.fi\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH AUTHOR\nThe Borg Collective\n.\\\" Generated by docutils manpage writer.\n.\n" }, { "path": "docs/man/borg-create.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-create\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-create \\- Creates a new archive.\n.SH SYNOPSIS\n.sp\nborg [common options] create [options] NAME [PATH...]\n.SH DESCRIPTION\n.sp\nThis command creates a backup archive containing all files found while recursively\ntraversing all specified paths. Paths are added to the archive as they are given,\nwhich means that if relative paths are desired, the command must be run from the correct\ndirectory.\n.sp\nThe slashdot hack in paths (recursion roots) is triggered by using \\fB/./\\fP:\n\\fB/this/gets/stripped/./this/gets/archived\\fP means to process that fs object, but\nstrip the prefix on the left side of \\fB\\&./\\fP from the archived items (in this case,\n\\fBthis/gets/archived\\fP will be the path in the archived item).\n.sp\nWhen specifying \\(aq\\-\\(aq as a path, borg will read data from standard input and create a\nfile named \\(aqstdin\\(aq in the created archive from that data. In some cases, it is more\nappropriate to use \\-\\-content\\-from\\-command. See the section \\fIReading from stdin\\fP\nbelow for details.\n.sp\nThe archive will consume almost no disk space for files or parts of files that\nhave already been stored in other archives.\n.sp\nThe \\fB\\-\\-tags\\fP option can be used to add a list of tags to the new archive.\n.sp\nThe archive name does not need to be unique; you can and should use the same\nname for a series of archives. The unique archive identifier is its ID (hash),\nand you can abbreviate the ID as long as it is unique.\n.sp\nIn the archive name, you may use the following placeholders:\n{now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.\n.sp\nBackup speed is increased by not reprocessing files that are already part of\nexisting archives and were not modified. The detection of unmodified files is\ndone by comparing multiple file metadata values with previous values kept in\nthe files cache.\n.sp\nThis comparison can operate in different modes as given by \\fB\\-\\-files\\-cache\\fP:\n.INDENT 0.0\n.IP \\(bu 2\nctime,size,inode (default)\n.IP \\(bu 2\nmtime,size,inode (default behaviour of borg versions older than 1.1.0rc4)\n.IP \\(bu 2\nctime,size (ignore the inode number)\n.IP \\(bu 2\nmtime,size (ignore the inode number)\n.IP \\(bu 2\nrechunk,ctime (all files are considered modified \\- rechunk, cache ctime)\n.IP \\(bu 2\nrechunk,mtime (all files are considered modified \\- rechunk, cache mtime)\n.IP \\(bu 2\ndisabled (disable the files cache, all files considered modified \\- rechunk)\n.UNINDENT\n.sp\ninode number: better safety, but often unstable on network filesystems\n.sp\nNormally, detecting file modifications will take inode information into\nconsideration to improve the reliability of file change detection.\nThis is problematic for files located on sshfs and similar network file\nsystems which do not provide stable inode numbers, such files will always\nbe considered modified. You can use modes without \\fIinode\\fP in this case to\nimprove performance, but reliability of change detection might be reduced.\n.sp\nctime vs. mtime: safety vs. speed\n.INDENT 0.0\n.IP \\(bu 2\nctime is a rather safe way to detect changes to a file (metadata and contents)\nas it cannot be set from userspace. But a metadata\\-only change will already\nupdate the ctime, so there might be some unnecessary chunking/hashing even\nwithout content changes. Some filesystems do not support ctime (change time).\nE.g. doing a chown or chmod to a file will change its ctime.\n.IP \\(bu 2\nmtime usually works and only updates if file contents were changed. But mtime\ncan be arbitrarily set from userspace, e.g., to set mtime back to the same value\nit had before a content change happened. This can be used maliciously as well as\nwell\\-meant, but in both cases mtime\\-based cache modes can be problematic.\n.UNINDENT\n.INDENT 0.0\n.TP\n.B The \\fB\\-\\-files\\-changed\\fP option controls how Borg detects if a file has changed during backup:\n.INDENT 7.0\n.IP \\(bu 2\nctime (default on POSIX): Use ctime to detect changes. This is the safest option.\nNot supported on Windows (ctime is file creation time there).\n.IP \\(bu 2\nmtime (default on Windows): Use mtime to detect changes.\n.IP \\(bu 2\ndisabled: Disable the \\(dqfile has changed while we backed it up\\(dq detection completely.\nThis is not recommended unless you know what you\\(aqre doing, as it could lead to\ninconsistent backups if files change during the backup process.\n.UNINDENT\n.UNINDENT\n.sp\nThe mount points of filesystems or filesystem snapshots should be the same for every\ncreation of a new archive to ensure fast operation. This is because the file cache that\nis used to determine changed files quickly uses absolute filenames.\nIf this is not possible, consider creating a bind mount to a stable location.\n.sp\nThe \\fB\\-\\-progress\\fP option shows (from left to right) Original and (uncompressed)\ndeduplicated size (O and U respectively), then the Number of files (N) processed so far,\nfollowed by the currently processed path.\n.sp\nWhen using \\fB\\-\\-stats\\fP, you will get some statistics about how much data was\nadded \\- the \\(dqThis Archive\\(dq deduplicated size there is most interesting as that is\nhow much your repository will grow. Please note that the \\(dqAll archives\\(dq stats refer to\nthe state after creation. Also, the \\fB\\-\\-stats\\fP and \\fB\\-\\-dry\\-run\\fP options are mutually\nexclusive because the data is not actually compressed and deduplicated during a dry run.\n.sp\nFor more help on include/exclude patterns, see the \\fIborg_patterns\\fP command output.\n.sp\nFor more help on placeholders, see the \\fIborg_placeholders\\fP command output.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.TP\n.B PATH\npaths to archive\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-n\\fP,\\fB \\-\\-dry\\-run\ndo not create a backup archive\n.TP\n.B \\-s\\fP,\\fB \\-\\-stats\nprint statistics for the created archive\n.TP\n.B \\-\\-list\noutput a verbose list of items (files, dirs, ...)\n.TP\n.BI \\-\\-filter \\ STATUSCHARS\nonly display items with the given status characters (see description)\n.TP\n.B \\-\\-json\noutput stats as JSON. Implies \\fB\\-\\-stats\\fP\\&.\n.TP\n.BI \\-\\-stdin\\-name \\ NAME\nuse NAME in archive for stdin data (default: \\(aqstdin\\(aq)\n.TP\n.BI \\-\\-stdin\\-user \\ USER\nset user USER in archive for stdin data (default: do not store user/uid)\n.TP\n.BI \\-\\-stdin\\-group \\ GROUP\nset group GROUP in archive for stdin data (default: do not store group/gid)\n.TP\n.BI \\-\\-stdin\\-mode \\ M\nset mode to M in archive for stdin data (default: 0660)\n.TP\n.B \\-\\-content\\-from\\-command\ninterpret PATH as a command and store its stdout. See also the section \\(aqReading from stdin\\(aq below.\n.TP\n.B \\-\\-paths\\-from\\-stdin\nread DELIM\\-separated list of paths to back up from stdin. All control is external: it will back up all files given \\- no more, no less.\n.TP\n.B \\-\\-paths\\-from\\-command\ninterpret PATH as command and treat its output as \\fB\\-\\-paths\\-from\\-stdin\\fP\n.TP\n.B \\-\\-paths\\-from\\-shell\\-command\ninterpret PATH as shell command and treat its output as \\fB\\-\\-paths\\-from\\-stdin\\fP\n.TP\n.BI \\-\\-paths\\-delimiter \\ DELIM\nset path delimiter for \\fB\\-\\-paths\\-from\\-stdin\\fP and \\fB\\-\\-paths\\-from\\-command\\fP (default: \\fB\\en\\fP)\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.TP\n.B \\-\\-exclude\\-caches\nexclude directories that contain a CACHEDIR.TAG file (\\%)\n.TP\n.BI \\-\\-exclude\\-if\\-present \\ NAME\nexclude directories that are tagged by containing a filesystem object with the given NAME\n.TP\n.B \\-\\-keep\\-exclude\\-tags\nif tag objects are specified with \\fB\\-\\-exclude\\-if\\-present\\fP, do not omit the tag objects themselves from the backup archive\n.UNINDENT\n.SS Filesystem options\n.INDENT 0.0\n.TP\n.B \\-x\\fP,\\fB \\-\\-one\\-file\\-system\nstay in the same file system and do not store mount points of other file systems \\- this might behave different from your expectations, see the description below.\n.TP\n.B \\-\\-numeric\\-ids\nonly store numeric user and group identifiers\n.TP\n.B \\-\\-atime\ndo store atime into archive\n.TP\n.B \\-\\-noctime\ndo not store ctime into archive\n.TP\n.B \\-\\-nobirthtime\ndo not store birthtime (creation date) into archive\n.TP\n.B \\-\\-noflags\ndo not read and store flags (e.g. NODUMP, IMMUTABLE) into archive\n.TP\n.B \\-\\-noacls\ndo not read and store ACLs into archive\n.TP\n.B \\-\\-noxattrs\ndo not read and store xattrs into archive\n.TP\n.B \\-\\-sparse\ndetect sparse holes in input (supported only by fixed chunker)\n.TP\n.BI \\-\\-files\\-cache \\ MODE\noperate files cache in MODE. default: ctime,size,inode\n.TP\n.BI \\-\\-files\\-changed \\ MODE\nspecify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).\n.TP\n.B \\-\\-read\\-special\nopen and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.\n.UNINDENT\n.SS Archive options\n.INDENT 0.0\n.TP\n.BI \\-\\-comment \\ COMMENT\nadd a comment text to the archive\n.TP\n.BI \\-\\-timestamp \\ TIMESTAMP\nmanually specify the archive creation date/time (yyyy\\-mm\\-ddThh:mm:ss[(+|\\-)HH:MM] format, (+|\\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory.\n.TP\n.BI \\-\\-chunker\\-params \\ PARAMS\nspecify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095\n.TP\n.BI \\-C \\ COMPRESSION\\fR,\\fB \\ \\-\\-compression \\ COMPRESSION\nselect compression algorithm, see the output of the \\(dqborg help compression\\(dq command for details.\n.TP\n.BI \\-\\-hostname \\ HOSTNAME\nexplicitly set hostname for the archive\n.TP\n.BI \\-\\-username \\ USERNAME\nexplicitly set username for the archive\n.TP\n.BI \\-\\-tags \\ TAG\nadd tags to archive (comma\\-separated or multiple arguments)\n.UNINDENT\n.SH EXAMPLES\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\nArchive series and performance: In Borg 2, archives that share the same NAME form an \\(dqarchive series\\(dq.\nThe files cache is maintained per series. For best performance on repeated backups, reuse the same\nNAME every time you run \\fBborg create\\fP for the same dataset (e.g. always use \\fBmy\\-documents\\fP).\nFrequently changing the NAME (for example by embedding date/time like \\fBmy\\-documents\\-2025\\-11\\-10\\fP)\nprevents cache reuse and forces Borg to re\\-scan and re\\-chunk files, which can make incremental\nbackups vastly slower. Only vary the NAME if you intentionally want to start a new series.\n.sp\nIf you must vary the archive name but still want cache reuse across names, see the advanced\nknobs described in \\fIupgradenotes2\\fP (\\fBBORG_FILES_CACHE_SUFFIX\\fP and \\fBBORG_FILES_CACHE_TTL\\fP),\nbut the recommended approach is to keep a stable NAME per series.\n.UNINDENT\n.UNINDENT\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Backup ~/Documents into an archive named \\(dqmy\\-documents\\(dq\n$ borg create my\\-documents ~/Documents\n\n# same, but list all files as we process them\n$ borg create \\-\\-list my\\-documents ~/Documents\n\n# Backup /mnt/disk/docs, but strip path prefix using the slashdot hack\n$ borg create \\-\\-repo /path/to/repo docs /mnt/disk/./docs\n\n# Backup ~/Documents and ~/src but exclude pyc files\n$ borg create my\\-files \\e\n ~/Documents \\e\n ~/src \\e\n \\-\\-exclude \\(aq*.pyc\\(aq\n\n# Backup home directories excluding image thumbnails (i.e. only\n# /home//.thumbnails is excluded, not /home/*/*/.thumbnails etc.)\n$ borg create my\\-files /home \\-\\-exclude \\(aqsh:home/*/.thumbnails\\(aq\n\n# Back up the root filesystem into an archive named \\(dqroot\\-archive\\(dq\n# Use zlib compression (good, but slow) — default is LZ4 (fast, low compression ratio)\n$ borg create \\-C zlib,6 \\-\\-one\\-file\\-system root\\-archive /\n\n# Backup into an archive name like FQDN\\-root\n$ borg create \\(aq{fqdn}\\-root\\(aq /\n\n# Back up a remote host locally (\\(dqpull\\(dq style) using SSHFS\n$ mkdir sshfs\\-mount\n$ sshfs root@example.com:/ sshfs\\-mount\n$ cd sshfs\\-mount\n$ borg create example.com\\-root .\n$ cd ..\n$ fusermount \\-u sshfs\\-mount\n\n# Make a big effort in fine\\-grained deduplication (big chunk management\n# overhead, needs a lot of RAM and disk space; see the formula in the internals docs):\n$ borg create \\-\\-chunker\\-params buzhash,10,23,16,4095 small /smallstuff\n\n# Backup a raw device (must not be active/in use/mounted at that time)\n$ borg create \\-\\-read\\-special \\-\\-chunker\\-params fixed,4194304 my\\-sdx /dev/sdX\n\n# Backup a sparse disk image (must not be active/in use/mounted at that time)\n$ borg create \\-\\-sparse \\-\\-chunker\\-params fixed,4194304 my\\-disk my\\-disk.raw\n\n# No compression (none)\n$ borg create \\-\\-compression none arch ~\n\n# Super fast, low compression (lz4, default)\n$ borg create arch ~\n\n# Less fast, higher compression (zlib, N = 0..9)\n$ borg create \\-\\-compression zlib,N arch ~\n\n# Even slower, even higher compression (lzma, N = 0..9)\n$ borg create \\-\\-compression lzma,N arch ~\n\n# Only compress compressible data with lzma,N (N = 0..9)\n$ borg create \\-\\-compression auto,lzma,N arch ~\n\n# Use the short hostname and username as the archive name\n$ borg create \\(aq{hostname}\\-{user}\\(aq ~\n\n# Back up relative paths by moving into the correct directory first\n$ cd /home/user/Documents\n# The root directory of the archive will be \\(dqprojectA\\(dq\n$ borg create \\(aqdaily\\-projectA\\(aq projectA\n\n# Use external command to determine files to archive\n# Use \\-\\-paths\\-from\\-stdin with find to back up only files less than 1 MB in size\n$ find ~ \\-size \\-1000k | borg create \\-\\-paths\\-from\\-stdin small\\-files\\-only\n# Use \\-\\-paths\\-from\\-command with find to back up files from only a given user\n$ borg create \\-\\-paths\\-from\\-command joes\\-files \\-\\- find /srv/samba/shared \\-user joe\n# Use \\-\\-paths\\-from\\-shell\\-command with find to back up a few files from only a given user \\-\n# BE VERY CAREFUL AND ONLY USE TRUSTED INPUT FOR THE SHELL COMMAND!\n$ borg create \\-\\-paths\\-from\\-shell\\-command some\\-of\\-joes\\-files \\-\\- \\(dqfind /srv/samba/shared \\-user joe | head\\(dq\n# Use \\-\\-paths\\-from\\-stdin with \\-\\-paths\\-delimiter (for example, for filenames with newlines in them)\n$ find ~ \\-size \\-1000k \\-print0 | borg create \\e\n \\-\\-paths\\-from\\-stdin \\e\n \\-\\-paths\\-delimiter \\(dq\\e0\\(dq \\e\n smallfiles\\-handle\\-newline\n.EE\n.UNINDENT\n.UNINDENT\n.SH NOTES\n.sp\nThe \\fB\\-\\-exclude\\fP patterns are not like tar. In tar \\fB\\-\\-exclude\\fP .bundler/gems will\nexclude foo/.bundler/gems. In borg it will not, you need to use \\fB\\-\\-exclude\\fP\n\\(aq*/.bundler/gems\\(aq to get the same effect.\n.sp\nIn addition to using \\fB\\-\\-exclude\\fP patterns, it is possible to use\n\\fB\\-\\-exclude\\-if\\-present\\fP to specify the name of a filesystem object (e.g. a file\nor folder name) which, when contained within another folder, will prevent the\ncontaining folder from being backed up. By default, the containing folder and\nall of its contents will be omitted from the backup. If, however, you wish to\nonly include the objects specified by \\fB\\-\\-exclude\\-if\\-present\\fP in your backup,\nand not include any other contents of the containing folder, this can be enabled\nthrough using the \\fB\\-\\-keep\\-exclude\\-tags\\fP option.\n.sp\nThe \\fB\\-x\\fP or \\fB\\-\\-one\\-file\\-system\\fP option excludes directories, that are mountpoints (and everything in them).\nIt detects mountpoints by comparing the device number from the output of \\fBstat()\\fP of the directory and its\nparent directory. Specifically, it excludes directories for which \\fBstat()\\fP reports a device number different\nfrom the device number of their parent.\nIn general: be aware that there are directories with device number different from their parent, which the kernel\ndoes not consider a mountpoint and also the other way around.\nLinux examples for this are bind mounts (possibly same device number, but always a mountpoint) and ALL\nsubvolumes of a btrfs (different device number from parent but not necessarily a mountpoint).\nmacOS examples are the apfs mounts of a typical macOS installation.\nTherefore, when using \\fB\\-\\-one\\-file\\-system\\fP, you should double\\-check that the backup works as intended.\n.SS Item flags\n.sp\n\\fB\\-\\-list\\fP outputs a list of all files, directories and other\nfile system items it considered (no matter whether they had content changes\nor not). For each item, it prefixes a single\\-letter flag that indicates type\nand/or status of the item.\n.sp\nIf you are interested only in a subset of that output, you can give e.g.\n\\fB\\-\\-filter=AME\\fP and it will only show regular files with A, M or E status (see\nbelow).\n.sp\nA uppercase character represents the status of a regular file relative to the\n\\(dqfiles\\(dq cache (not relative to the repo \\-\\- this is an issue if the files cache\nis not used). Metadata is stored in any case and for \\(aqA\\(aq and \\(aqM\\(aq also new data\nchunks are stored. For \\(aqU\\(aq all data chunks refer to already existing chunks.\n.INDENT 0.0\n.IP \\(bu 2\n\\(aqA\\(aq = regular file, added (see also \\fIa_status_oddity\\fP in the FAQ)\n.IP \\(bu 2\n\\(aqM\\(aq = regular file, modified\n.IP \\(bu 2\n\\(aqU\\(aq = regular file, unchanged\n.IP \\(bu 2\n\\(aqC\\(aq = regular file, it changed while we backed it up\n.IP \\(bu 2\n\\(aqE\\(aq = regular file, an error happened while accessing/reading \\fIthis\\fP file\n.UNINDENT\n.sp\nA lowercase character means a file type other than a regular file,\nborg usually just stores their metadata:\n.INDENT 0.0\n.IP \\(bu 2\n\\(aqd\\(aq = directory\n.IP \\(bu 2\n\\(aqb\\(aq = block device\n.IP \\(bu 2\n\\(aqc\\(aq = char device\n.IP \\(bu 2\n\\(aqh\\(aq = regular file, hard link (to already seen inodes)\n.IP \\(bu 2\n\\(aqs\\(aq = symlink\n.IP \\(bu 2\n\\(aqf\\(aq = fifo\n.UNINDENT\n.sp\nOther flags used include:\n.INDENT 0.0\n.IP \\(bu 2\n\\(aq+\\(aq = included, item would be backed up (if not in dry\\-run mode)\n.IP \\(bu 2\n\\(aq\\-\\(aq = excluded, item would not be / was not backed up\n.IP \\(bu 2\n\\(aqi\\(aq = backup data was read from standard input (stdin)\n.IP \\(bu 2\n\\(aq?\\(aq = missing status code (if you see this, please file a bug report!)\n.UNINDENT\n.SS Reading backup data from stdin\n.sp\nThere are two methods to read from stdin. Either specify \\fB\\-\\fP as path and\npipe directly to borg:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nbackup\\-vm \\-\\-id myvm \\-\\-stdout | borg create \\-\\-repo REPO ARCHIVE \\-\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nOr use \\fB\\-\\-content\\-from\\-command\\fP to have Borg manage the execution of the\ncommand and piping. If you do so, the first PATH argument is interpreted\nas command to execute and any further arguments are treated as arguments\nto the command:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nborg create \\-\\-content\\-from\\-command \\-\\-repo REPO ARCHIVE \\-\\- backup\\-vm \\-\\-id myvm \\-\\-stdout\n.EE\n.UNINDENT\n.UNINDENT\n.sp\n\\fB\\-\\-\\fP is used to ensure \\fB\\-\\-id\\fP and \\fB\\-\\-stdout\\fP are \\fBnot\\fP considered\narguments to \\fBborg\\fP but rather \\fBbackup\\-vm\\fP\\&.\n.sp\nThe difference between the two approaches is that piping to borg creates an\narchive even if the command piping to borg exits with a failure. In this case,\n\\fBone can end up with truncated output being backed up\\fP\\&. Using\n\\fB\\-\\-content\\-from\\-command\\fP, in contrast, borg is guaranteed to fail without\ncreating an archive should the command fail. The command is considered failed\nwhen it returned a non\\-zero exit code.\n.sp\nReading from stdin yields just a stream of data without file metadata\nassociated with it, and the files cache is not needed at all. So it is\nsafe to disable it via \\fB\\-\\-files\\-cache disabled\\fP and speed up backup\ncreation a bit.\n.sp\nBy default, the content read from stdin is stored in a file called \\(aqstdin\\(aq.\nUse \\fB\\-\\-stdin\\-name\\fP to change the name.\n.SS Feeding all file paths from externally\n.sp\nUsually, you give a starting path (recursion root) to borg and then borg\nautomatically recurses, finds and backs up all fs objects contained in\nthere (optionally considering include/exclude rules).\n.sp\nIf you need more control and you want to give every single fs object path\nto borg (maybe implementing your own recursion or your own rules), you can use\n\\fB\\-\\-paths\\-from\\-stdin\\fP, \\fB\\-\\-paths\\-from\\-command\\fP or \\fB\\-\\-paths\\-from\\-shell\\-command\\fP\n(with the latter two, borg will fail to create an archive should the command fail).\n.sp\nBorg supports paths with the slashdot hack to strip path prefixes here also.\nSo, be careful not to unintentionally trigger that.\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-delete(1)\\fP, \\fIborg\\-prune(1)\\fP, \\fIborg\\-check(1)\\fP, \\fIborg\\-patterns(1)\\fP, \\fIborg\\-placeholders(1)\\fP, \\fIborg\\-compression(1)\\fP, \\fIborg\\-repo\\-create(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-delete.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-delete\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-delete \\- Deletes archives.\n.SH SYNOPSIS\n.sp\nborg [common options] delete [options] [NAME]\n.SH DESCRIPTION\n.sp\nThis command soft\\-deletes archives from the repository.\n.sp\nImportant:\n.INDENT 0.0\n.IP \\(bu 2\nThe delete command will only mark archives for deletion (\\(dqsoft\\-deletion\\(dq),\nrepository disk space is \\fBnot\\fP freed until you run \\fBborg compact\\fP\\&.\n.IP \\(bu 2\nYou can use \\fBborg undelete\\fP to undelete archives, but only until\nyou run \\fBborg compact\\fP\\&.\n.UNINDENT\n.sp\nWhen in doubt, use \\fB\\-\\-dry\\-run \\-\\-list\\fP to see what would be deleted.\n.sp\nYou can delete multiple archives by specifying a match pattern using\nthe \\fB\\-\\-match\\-archives PATTERN\\fP option (for more information on these\npatterns, see \\fIborg_patterns\\fP).\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-n\\fP,\\fB \\-\\-dry\\-run\ndo not change the repository\n.TP\n.B \\-\\-list\noutput a verbose list of archives\n.UNINDENT\n.SS Archive filters\n.INDENT 0.0\n.TP\n.BI \\-a \\ PATTERN\\fR,\\fB \\ \\-\\-match\\-archives \\ PATTERN\nonly consider archives matching all patterns. See \\(dqborg help match\\-archives\\(dq.\n.TP\n.BI \\-\\-sort\\-by \\ KEYS\nComma\\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp\n.TP\n.BI \\-\\-first \\ N\nconsider the first N archives after other filters are applied\n.TP\n.BI \\-\\-last \\ N\nconsider the last N archives after other filters are applied\n.TP\n.BI \\-\\-oldest \\ TIMESPAN\nconsider archives between the oldest archive\\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newest \\ TIMESPAN\nconsider archives between the newest archive\\(aqs timestamp and (newest \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-older \\ TIMESPAN\nconsider archives older than (now \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newer \\ TIMESPAN\nconsider archives newer than (now \\- TIMESPAN), e.g., 7d or 12m.\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Delete all backup archives named \\(dqkenny\\-files\\(dq:\n$ borg delete \\-a kenny\\-files\n# Actually free disk space:\n$ borg compact\n\n# Delete a specific backup archive using its unique archive ID prefix\n$ borg delete aid:d34db33f\n\n# Delete all archives whose names begin with the machine\\(aqs hostname followed by \\(dq\\-\\(dq\n$ borg delete \\-a \\(aqsh:{hostname}\\-*\\(aq\n\n# Delete all archives whose names contain \\(dq\\-2012\\-\\(dq\n$ borg delete \\-a \\(aqsh:*\\-2012\\-*\\(aq\n\n# See what would be deleted if delete was run without \\-\\-dry\\-run\n$ borg delete \\-\\-list \\-\\-dry\\-run \\-a \\(aqsh:*\\-May\\-*\\(aq\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-compact(1)\\fP, \\fIborg\\-repo\\-delete(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-diff.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-diff\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-diff \\- Finds differences between two archives.\n.SH SYNOPSIS\n.sp\nborg [common options] diff [options] ARCHIVE1 ARCHIVE2 [PATH...]\n.SH DESCRIPTION\n.sp\nThis command finds differences (file contents, metadata) between ARCHIVE1 and ARCHIVE2.\n.sp\nFor more help on include/exclude patterns, see the output of the \\fIborg_patterns\\fP command.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B ARCHIVE1\nARCHIVE1 name\n.TP\n.B ARCHIVE2\nARCHIVE2 name\n.TP\n.B PATH\npaths of items inside the archives to compare; patterns are supported.\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-numeric\\-ids\nonly consider numeric user and group identifiers\n.TP\n.B \\-\\-same\\-chunker\\-params\noverride the check of chunker parameters\n.TP\n.BI \\-\\-format \\ FORMAT\nspecify format for differences between archives (default: \\(dq{change} {path}{NL}\\(dq)\n.TP\n.B \\-\\-json\\-lines\nFormat output as JSON Lines.\n.TP\n.B \\-\\-sort\\-by\nSort output by comma\\-separated fields (e.g., \\(aq>size_added,path\\(aq).\n.TP\n.B \\-\\-content\\-only\nOnly compare differences in content (exclude metadata differences)\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ borg diff archive1 archive2\n +17 B \\-5 B [\\-rw\\-r\\-\\-r\\-\\- \\-> \\-rwxr\\-xr\\-x] file1\n +135 B \\-252 B file2\nadded 0 B file4\nremoved 0 B file3\n\n$ borg diff archive1 archive2\n{\\(dqpath\\(dq: \\(dqfile1\\(dq, \\(dqchanges\\(dq: [{\\(dqtype\\(dq: \\(dqmodified\\(dq, \\(dqadded\\(dq: 17, \\(dqremoved\\(dq: 5}, {\\(dqtype\\(dq: \\(dqmode\\(dq, \\(dqold_mode\\(dq: \\(dq\\-rw\\-r\\-\\-r\\-\\-\\(dq, \\(dqnew_mode\\(dq: \\(dq\\-rwxr\\-xr\\-x\\(dq}]}\n{\\(dqpath\\(dq: \\(dqfile2\\(dq, \\(dqchanges\\(dq: [{\\(dqtype\\(dq: \\(dqmodified\\(dq, \\(dqadded\\(dq: 135, \\(dqremoved\\(dq: 252}]}\n{\\(dqpath\\(dq: \\(dqfile4\\(dq, \\(dqchanges\\(dq: [{\\(dqtype\\(dq: \\(dqadded\\(dq, \\(dqsize\\(dq: 0}]}\n{\\(dqpath\\(dq: \\(dqfile3\\(dq, \\(dqchanges\\(dq: [{\\(dqtype\\(dq: \\(dqremoved\\(dq, \\(dqsize\\(dq: 0}]}\n\n\n# Use \\-\\-sort\\-by with a comma\\-separated list; sorts apply stably from last to first.\n# Here: primary by net size change descending, tie\\-breaker by path ascending\n$ borg diff \\-\\-sort\\-by=\\(dq>size_diff,path\\(dq archive1 archive2\n +17 B \\-5 B [\\-rw\\-r\\-\\-r\\-\\- \\-> \\-rwxr\\-xr\\-x] file1\nremoved 0 B file3\nadded 0 B file4\n +135 B \\-252 B file2\n.EE\n.UNINDENT\n.UNINDENT\n.SH NOTES\n.SS The FORMAT specifier syntax\n.sp\nThe \\fB\\-\\-format\\fP option uses Python\\(aqs format string syntax \\%\\&.\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ borg diff \\-\\-format \\(aq{content:30} {path}{NL}\\(aq ArchiveFoo ArchiveBar\nmodified: +4.1 kB \\-1.0 kB file\\-diff\n\\&...\n\n# {VAR:NUMBER} \\- pad to NUMBER columns right\\-aligned.\n$ borg diff \\-\\-format \\(aq{content:>30} {path}{NL}\\(aq ArchiveFoo ArchiveBar\n modified: +4.1 kB \\-1.0 kB file\\-diff\n\\&...\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nThe following keys are always available:\n\\- NEWLINE: OS dependent line separator\n\\- NL: alias of NEWLINE\n\\- NUL: NUL character for creating print0 / xargs \\-0 like output\n\\- SPACE: space character\n\\- TAB: tab character\n\\- CR: carriage return character\n\\- LF: line feed character\n.sp\nKeys available only when showing differences between archives:\n.INDENT 0.0\n.IP \\(bu 2\npath: archived file path\n.IP \\(bu 2\nchange: all available changes\n.IP \\(bu 2\ncontent: file content change\n.IP \\(bu 2\nmode: file mode change\n.IP \\(bu 2\ntype: file type change\n.IP \\(bu 2\nowner: file owner (user/group) change\n.IP \\(bu 2\ngroup: file group change\n.IP \\(bu 2\nuser: file user change\n.IP \\(bu 2\nlink: file link change\n.IP \\(bu 2\ndirectory: file directory change\n.IP \\(bu 2\nblkdev: file block device change\n.IP \\(bu 2\nchrdev: file character device change\n.IP \\(bu 2\nfifo: file fifo change\n.IP \\(bu 2\nmtime: file modification time change\n.IP \\(bu 2\nctime: file change time change\n.IP \\(bu 2\nisomtime: file modification time change (ISO 8601)\n.IP \\(bu 2\nisoctime: file creation time change (ISO 8601)\n.UNINDENT\n.SS What is compared\n.sp\nFor each matching item in both archives, Borg reports:\n.INDENT 0.0\n.IP \\(bu 2\nContent changes: total added/removed bytes within files. If chunker parameters are comparable,\nBorg compares chunk IDs quickly; otherwise, it compares the content.\n.IP \\(bu 2\nMetadata changes: user, group, mode, and other metadata shown inline, like\n\\(dq[old_mode \\-> new_mode]\\(dq for mode changes. Use \\fB\\-\\-content\\-only\\fP to suppress metadata changes.\n.IP \\(bu 2\nAdded/removed items: printed as \\(dqadded SIZE path\\(dq or \\(dqremoved SIZE path\\(dq.\n.UNINDENT\n.SS Output formats\n.sp\nThe default (text) output shows one line per changed path, e.g.:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n+135 B \\-252 B [ \\-rw\\-r\\-\\-r\\-\\- \\-> \\-rwxr\\-xr\\-x ] path/to/file\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nJSON Lines output (\\fB\\-\\-json\\-lines\\fP) prints one JSON object per changed path, e.g.:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n{\\(dqpath\\(dq: \\(dqPATH\\(dq, \\(dqchanges\\(dq: [\n {\\(dqtype\\(dq: \\(dqmodified\\(dq, \\(dqadded\\(dq: BYTES, \\(dqremoved\\(dq: BYTES},\n {\\(dqtype\\(dq: \\(dqmode\\(dq, \\(dqold_mode\\(dq: \\(dq\\-rw\\-r\\-\\-r\\-\\-\\(dq, \\(dqnew_mode\\(dq: \\(dq\\-rwxr\\-xr\\-x\\(dq},\n {\\(dqtype\\(dq: \\(dqadded\\(dq, \\(dqsize\\(dq: SIZE},\n {\\(dqtype\\(dq: \\(dqremoved\\(dq, \\(dqsize\\(dq: SIZE}\n]}\n.EE\n.UNINDENT\n.UNINDENT\n.SS Sorting\n.sp\nUse \\fB\\-\\-sort\\-by FIELDS\\fP where FIELDS is a comma\\-separated list of fields.\nSorts are applied stably from last to first in the given list. Prepend \\(dq>\\(dq for\ndescending, \\(dq<\\(dq (or no prefix) for ascending, for example \\fB\\-\\-sort\\-by=\\(dq>size_added,path\\(dq\\fP\\&.\nSupported fields include:\n.INDENT 0.0\n.IP \\(bu 2\npath: the item path\n.IP \\(bu 2\nsize_added: total bytes added for the item content\n.IP \\(bu 2\nsize_removed: total bytes removed for the item content\n.IP \\(bu 2\nsize_diff: size_added \\- size_removed (net content change)\n.IP \\(bu 2\nsize: size of the item as stored in ARCHIVE2 (0 for removed items)\n.IP \\(bu 2\nuser, group, uid, gid, ctime, mtime: taken from the item state in ARCHIVE2 when present\n.IP \\(bu 2\nctime_diff, mtime_diff: timestamp difference (ARCHIVE2 \\- ARCHIVE1)\n.UNINDENT\n.SS Performance considerations\n.sp\ndiff automatically detects whether the archives were created with the same chunker\nparameters. If so, only chunk IDs are compared, which is very fast.\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-export-tar.1", "content": "'\\\" t\n.\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-export-tar\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-export-tar \\- Export archive contents as a tarball\n.SH SYNOPSIS\n.sp\nborg [common options] export\\-tar [options] NAME FILE [PATH...]\n.SH DESCRIPTION\n.sp\nThis command creates a tarball from an archive.\n.sp\nWhen giving \\(aq\\-\\(aq as the output FILE, Borg will write a tar stream to standard output.\n.sp\nBy default (\\fB\\-\\-tar\\-filter=auto\\fP) Borg will detect whether the FILE should be compressed\nbased on its file extension and pipe the tarball through an appropriate filter\nbefore writing it to FILE:\n.INDENT 0.0\n.IP \\(bu 2\n\\&.tar.gz or .tgz: gzip\n.IP \\(bu 2\n\\&.tar.bz2 or .tbz: bzip2\n.IP \\(bu 2\n\\&.tar.xz or .txz: xz\n.IP \\(bu 2\n\\&.tar.zstd or .tar.zst: zstd\n.IP \\(bu 2\n\\&.tar.lz4: lz4\n.UNINDENT\n.sp\nAlternatively, a \\fB\\-\\-tar\\-filter\\fP program may be explicitly specified. It should\nread the uncompressed tar stream from stdin and write a compressed/filtered\ntar stream to stdout.\n.sp\nDepending on the \\fB\\-\\-tar\\-format\\fP option, these formats are created:\n.TS\nbox center;\nl|l|l.\nT{\n\\-\\-tar\\-format\nT}\tT{\nSpecification\nT}\tT{\nMetadata\nT}\n_\nT{\nBORG\nT}\tT{\nBORG specific, like PAX\nT}\tT{\nall as supported by borg\nT}\n_\nT{\nPAX\nT}\tT{\nPOSIX.1\\-2001 (pax) format\nT}\tT{\nGNU + atime/ctime/mtime ns\n+ xattrs\nT}\n_\nT{\nGNU\nT}\tT{\nGNU tar format\nT}\tT{\nmtime s, no atime/ctime,\nno ACLs/xattrs/bsdflags\nT}\n.TE\n.sp\nA \\fB\\-\\-sparse\\fP option (as found in borg extract) is not supported.\n.sp\nBy default the entire archive is extracted but a subset of files and directories\ncan be selected by passing a list of \\fBPATHs\\fP as arguments.\nThe file selection can further be restricted by using the \\fB\\-\\-exclude\\fP option.\n.sp\nFor more help on include/exclude patterns, see the \\fIborg_patterns\\fP command output.\n.sp\n\\fB\\-\\-progress\\fP can be slower than no progress display, since it makes one additional\npass over the archive metadata.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.TP\n.B FILE\noutput tar file. \\(dq\\-\\(dq to write to stdout instead.\n.TP\n.B PATH\npaths to extract; patterns are supported\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-tar\\-filter\nfilter program to pipe data through\n.TP\n.B \\-\\-list\noutput verbose list of items (files, dirs, ...)\n.TP\n.BI \\-\\-tar\\-format \\ FMT\nselect tar format: BORG, PAX or GNU\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.TP\n.BI \\-\\-strip\\-components \\ NUMBER\nRemove the specified number of leading path elements. Paths with fewer elements will be silently skipped.\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-extract.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-extract\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-extract \\- Extracts archive contents.\n.SH SYNOPSIS\n.sp\nborg [common options] extract [options] NAME [PATH...]\n.SH DESCRIPTION\n.sp\nThis command extracts the contents of an archive.\n.sp\nBy default, the entire archive is extracted, but a subset of files and directories\ncan be selected by passing a list of \\fBPATH\\fP arguments. The default interpretation\nfor the paths to extract is \\fIpp:\\fP which is a literal path\\-prefix match. If you want\nto use e.g. a wildcard, you must select a different pattern style such as \\fIsh:\\fP or\n\\fIfm:\\fP\\&. See \\fIborg_patterns\\fP for more information.\n.sp\nThe file selection can be further restricted by using the \\fB\\-\\-exclude\\fP option.\nFor more help on include/exclude patterns, see the \\fIborg_patterns\\fP command output.\n.sp\nBy using \\fB\\-\\-dry\\-run\\fP, you can do all extraction steps except actually writing the\noutput data: reading metadata and data chunks from the repository, checking the hash/HMAC,\ndecrypting, and decompressing.\n.sp\n\\fB\\-\\-progress\\fP can be slower than no progress display, since it makes one additional\npass over the archive metadata.\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\nCurrently, extract always writes into the current working directory (\\(dq.\\(dq),\nso make sure you \\fBcd\\fP to the right place before calling \\fBborg extract\\fP\\&.\n.sp\nWhen parent directories are not extracted (because of using file/directory selection\nor any other reason), Borg cannot restore parent directories\\(aq metadata, e.g., owner,\ngroup, permissions, etc.\n.UNINDENT\n.UNINDENT\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.TP\n.B PATH\npaths to extract; patterns are supported\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-list\noutput a verbose list of items (files, dirs, ...)\n.TP\n.B \\-n\\fP,\\fB \\-\\-dry\\-run\ndo not actually change any files\n.TP\n.B \\-\\-numeric\\-ids\nonly use numeric user and group identifiers\n.TP\n.B \\-\\-noflags\ndo not extract/set flags (e.g. NODUMP, IMMUTABLE)\n.TP\n.B \\-\\-noacls\ndo not extract/set ACLs\n.TP\n.B \\-\\-noxattrs\ndo not extract/set xattrs\n.TP\n.B \\-\\-stdout\nwrite all extracted data to stdout\n.TP\n.B \\-\\-sparse\ncreate holes in the output sparse file from all\\-zero chunks\n.TP\n.B \\-\\-continue\ncontinue a previously interrupted extraction of the same archive\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.TP\n.BI \\-\\-strip\\-components \\ NUMBER\nRemove the specified number of leading path elements. Paths with fewer elements will be silently skipped.\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Extract entire archive\n$ borg extract my\\-files\n\n# Extract entire archive and list files while processing\n$ borg extract \\-\\-list my\\-files\n\n# Verify whether an archive could be successfully extracted, but do not write files to disk\n$ borg extract \\-\\-dry\\-run my\\-files\n\n# Extract the \\(dqsrc\\(dq directory\n$ borg extract my\\-files home/USERNAME/src\n\n# Extract the \\(dqsrc\\(dq directory but exclude object files\n$ borg extract my\\-files home/USERNAME/src \\-\\-exclude \\(aq*.o\\(aq\n\n# Extract only the C files\n$ borg extract my\\-files \\(aqsh:home/USERNAME/src/*.c\\(aq\n\n# Restore a raw device (must not be active/in use/mounted at that time)\n$ borg extract \\-\\-stdout my\\-sdx | dd of=/dev/sdx bs=10M\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-mount(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-import-tar.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-import-tar\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-import-tar \\- Create a backup archive from a tarball\n.SH SYNOPSIS\n.sp\nborg [common options] import\\-tar [options] NAME TARFILE\n.SH DESCRIPTION\n.sp\nThis command creates a backup archive from a tarball.\n.sp\nWhen giving \\(aq\\-\\(aq as path, Borg will read a tar stream from standard input.\n.sp\nBy default (\\-\\-tar\\-filter=auto) Borg will detect whether the file is compressed\nbased on its file extension and pipe the file through an appropriate filter:\n.INDENT 0.0\n.IP \\(bu 2\n\\&.tar.gz or .tgz: gzip \\-d\n.IP \\(bu 2\n\\&.tar.bz2 or .tbz: bzip2 \\-d\n.IP \\(bu 2\n\\&.tar.xz or .txz: xz \\-d\n.IP \\(bu 2\n\\&.tar.zstd or .tar.zst: zstd \\-d\n.IP \\(bu 2\n\\&.tar.lz4: lz4 \\-d\n.UNINDENT\n.sp\nAlternatively, a \\-\\-tar\\-filter program may be explicitly specified. It should\nread compressed data from stdin and output an uncompressed tar stream on\nstdout.\n.sp\nMost documentation of borg create applies. Note that this command does not\nsupport excluding files.\n.sp\nA \\fB\\-\\-sparse\\fP option (as found in borg create) is not supported.\n.sp\nAbout tar formats and metadata conservation or loss, please see \\fBborg export\\-tar\\fP\\&.\n.sp\nimport\\-tar reads these tar formats:\n.INDENT 0.0\n.IP \\(bu 2\nBORG: borg specific (PAX\\-based)\n.IP \\(bu 2\nPAX: POSIX.1\\-2001\n.IP \\(bu 2\nGNU: GNU tar\n.IP \\(bu 2\nPOSIX.1\\-1988 (ustar)\n.IP \\(bu 2\nUNIX V7 tar\n.IP \\(bu 2\nSunOS tar with extended attributes\n.UNINDENT\n.sp\nTo import multiple tarballs into a single archive, they can be simply\nconcatenated (e.g. using \\(dqcat\\(dq) into a single file, and imported with an\n\\fB\\-\\-ignore\\-zeros\\fP option to skip through the stop markers between them.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.TP\n.B TARFILE\ninput tar file. \\(dq\\-\\(dq to read from stdin instead.\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-tar\\-filter\nfilter program to pipe data through\n.TP\n.B \\-s\\fP,\\fB \\-\\-stats\nprint statistics for the created archive\n.TP\n.B \\-\\-list\noutput verbose list of items (files, dirs, ...)\n.TP\n.BI \\-\\-filter \\ STATUSCHARS\nonly display items with the given status characters\n.TP\n.B \\-\\-json\noutput stats as JSON (implies \\-\\-stats)\n.TP\n.B \\-\\-ignore\\-zeros\nignore zero\\-filled blocks in the input tarball\n.UNINDENT\n.SS Archive options\n.INDENT 0.0\n.TP\n.BI \\-\\-comment \\ COMMENT\nadd a comment text to the archive\n.TP\n.BI \\-\\-timestamp \\ TIMESTAMP\nmanually specify the archive creation date/time (yyyy\\-mm\\-ddThh:mm:ss[(+|\\-)HH:MM] format, (+|\\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory.\n.TP\n.BI \\-\\-chunker\\-params \\ PARAMS\nspecify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095\n.TP\n.BI \\-C \\ COMPRESSION\\fR,\\fB \\ \\-\\-compression \\ COMPRESSION\nselect compression algorithm, see the output of the \\(dqborg help compression\\(dq command for details.\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Export as an uncompressed tar archive\n$ borg export\\-tar Monday Monday.tar\n\n# Import an uncompressed tar archive\n$ borg import\\-tar Monday Monday.tar\n\n# Exclude some file types and compress using gzip\n$ borg export\\-tar Monday Monday.tar.gz \\-\\-exclude \\(aq*.so\\(aq\n\n# Use a higher compression level with gzip\n$ borg export\\-tar \\-\\-tar\\-filter=\\(dqgzip \\-9\\(dq Monday Monday.tar.gz\n\n# Copy an archive from repoA to repoB\n$ borg \\-r repoA export\\-tar \\-\\-tar\\-format=BORG archive \\- | borg \\-r repoB import\\-tar archive \\-\n\n# Export a tar, but instead of storing it on disk, upload it to a remote site using curl\n$ borg export\\-tar Monday \\- | curl \\-\\-data\\-binary @\\- https://somewhere/to/POST\n\n# Remote extraction via \\(aqtarpipe\\(aq\n$ borg export\\-tar Monday \\- | ssh somewhere \\(dqcd extracted; tar x\\(dq\n.EE\n.UNINDENT\n.UNINDENT\n.SS Archives transfer script\n.sp\nOutputs a script that copies all archives from repo1 to repo2:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nfor N I T in \\(gaborg list \\-\\-format=\\(aq{archive} {id} {time:%Y\\-%m\\-%dT%H:%M:%S}{NL}\\(aq\\(ga\ndo\n echo \\(dqborg \\-r repo1 export\\-tar \\-\\-tar\\-format=BORG aid:$I \\- | borg \\-r repo2 import\\-tar \\-\\-timestamp=$T $N \\-\\(dq\ndone\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nKept:\n.INDENT 0.0\n.IP \\(bu 2\narchive name, archive timestamp\n.IP \\(bu 2\narchive contents (all items with metadata and data)\n.UNINDENT\n.sp\nLost:\n.INDENT 0.0\n.IP \\(bu 2\nsome archive metadata (like the original command line, execution time, etc.)\n.UNINDENT\n.sp\nPlease note:\n.INDENT 0.0\n.IP \\(bu 2\nall data goes over that pipe, again and again for every archive\n.IP \\(bu 2\nthe pipe is dumb, there is no data or transfer time reduction there due to deduplication\n.IP \\(bu 2\nmaybe add compression\n.IP \\(bu 2\npipe over ssh for remote transfer\n.IP \\(bu 2\nno special sparse file support\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-info.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-info\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-info \\- Show archive details such as disk space used\n.SH SYNOPSIS\n.sp\nborg [common options] info [options] [NAME]\n.SH DESCRIPTION\n.sp\nThis command displays detailed information about the specified archive.\n.sp\nPlease note that the deduplicated sizes of the individual archives do not add\nup to the deduplicated size of the repository (\\(dqall archives\\(dq), because the two\nmean different things:\n.sp\nThis archive / deduplicated size = amount of data stored ONLY for this archive\n= unique chunks of this archive.\nAll archives / deduplicated size = amount of data stored in the repository\n= all chunks in the repository.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-json\nformat output as JSON\n.UNINDENT\n.SS Archive filters\n.INDENT 0.0\n.TP\n.BI \\-a \\ PATTERN\\fR,\\fB \\ \\-\\-match\\-archives \\ PATTERN\nonly consider archives matching all patterns. See \\(dqborg help match\\-archives\\(dq.\n.TP\n.BI \\-\\-sort\\-by \\ KEYS\nComma\\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp\n.TP\n.BI \\-\\-first \\ N\nconsider the first N archives after other filters are applied\n.TP\n.BI \\-\\-last \\ N\nconsider the last N archives after other filters are applied\n.TP\n.BI \\-\\-oldest \\ TIMESPAN\nconsider archives between the oldest archive\\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newest \\ TIMESPAN\nconsider archives between the newest archive\\(aqs timestamp and (newest \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-older \\ TIMESPAN\nconsider archives older than (now \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newer \\ TIMESPAN\nconsider archives newer than (now \\- TIMESPAN), e.g., 7d or 12m.\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ borg info aid:f7dea078\nArchive name: source\\-backup\nArchive fingerprint: f7dea0788dfc026cc2be1c0f5b94beb4e4084eb3402fc40c38d8719b1bf2d943\nComment:\nHostname: mba2020\nUsername: tw\nTime (start): Sat, 2022\\-06\\-25 20:51:40\nTime (end): Sat, 2022\\-06\\-25 20:51:40\nDuration: 0.03 seconds\nCommand line: /usr/bin/borg \\-r path/to/repo create source\\-backup src\nUtilization of maximum supported archive size: 0%\nNumber of files: 244\nOriginal size: 13.80 MB\nDeduplicated size: 531 B\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-list(1)\\fP, \\fIborg\\-diff(1)\\fP, \\fIborg\\-repo\\-info(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-key-change-algorithm.1", "content": ".\\\" Man page generated from reStructuredText.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"BORG-KEY-CHANGE-ALGORITHM\" 1 \"2022-06-26\" \"\" \"borg backup tool\"\n.SH NAME\nborg-key-change-algorithm \\- Change repository key algorithm\n.SH SYNOPSIS\n.sp\nborg [common options] key change\\-algorithm [options] ALGORITHM\n.SH DESCRIPTION\n.sp\nChange the algorithm we use to encrypt and authenticate the borg key.\n.sp\nImportant: In a \\fIrepokey\\fP mode (e.g. repokey\\-blake2) all users share the same key.\nIn this mode upgrading to \\fIargon2\\fP will make it impossible to access the repo for users who use an old version of borg.\nWe recommend upgrading to the latest stable version.\n.sp\nImportant: In a \\fIkeyfile\\fP mode (e.g. keyfile\\-blake2) each user has their own key (in \\fB~/.config/borg/keys\\fP).\nIn this mode this command will only change the key used by the current user.\nIf you want to upgrade to \\fIargon2\\fP to strengthen security, you will have to upgrade each user\\(aqs key individually.\n.sp\nYour repository is encrypted and authenticated with a key that is randomly generated by \\fBborg init\\fP\\&.\nThe key is encrypted and authenticated with your passphrase.\n.sp\nWe currently support two choices:\n.INDENT 0.0\n.IP 1. 3\nargon2 \\- recommended. This algorithm is used by default when initialising a new repository.\nThe key encryption key is derived from your passphrase via argon2\\-id.\nArgon2 is considered more modern and secure than pbkdf2.\n.IP 2. 3\npbkdf2 \\- the legacy algorithm. Use this if you want to access your repo via old versions of borg.\nThe key encryption key is derived from your passphrase via PBKDF2\\-HMAC\\-SHA256.\n.UNINDENT\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.nf\n.ft C\n# Upgrade an existing key to argon2\nborg key change\\-algorithm /path/to/repo argon2\n# Downgrade to pbkdf2 \\- use this if upgrading borg is not an option\nborg key change\\-algorithm /path/to/repo pbkdf2\n.ft P\n.fi\n.UNINDENT\n.UNINDENT\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B ALGORITHM\nselect key algorithm\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH AUTHOR\nThe Borg Collective\n.\\\" Generated by docutils manpage writer.\n.\n" }, { "path": "docs/man/borg-key-change-location.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-key-change-location\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-key-change-location \\- Changes the repository key location.\n.SH SYNOPSIS\n.sp\nborg [common options] key change\\-location [options] KEY_LOCATION\n.SH DESCRIPTION\n.sp\nChange the location of a Borg key. The key can be stored at different locations:\n.INDENT 0.0\n.IP \\(bu 2\nkeyfile: locally, usually in the home directory\n.IP \\(bu 2\nrepokey: inside the repository (in the repository config)\n.UNINDENT\n.sp\nPlease note:\n.sp\nThis command does NOT change the crypto algorithms, just the key location,\nthus you must ONLY give the key location (keyfile or repokey).\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B KEY_LOCATION\nselect key location\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-keep\nkeep the key also at the current location (default: remove it)\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-key-change-passphrase.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-key-change-passphrase\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-key-change-passphrase \\- Changes the repository key file passphrase.\n.SH SYNOPSIS\n.sp\nborg [common options] key change\\-passphrase [options]\n.SH DESCRIPTION\n.sp\nThe key files used for repository encryption are optionally passphrase\nprotected. This command can be used to change this passphrase.\n.sp\nPlease note that this command only changes the passphrase, but not any\nsecret protected by it (like e.g. encryption/MAC keys or chunker seed).\nThus, changing the passphrase after passphrase and borg key got compromised\ndoes not protect future (nor past) backups to the same repository.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Create a key file protected repository\n$ borg repo\\-create \\-\\-encryption=keyfile\\-aes\\-ocb \\-v\nInitializing repository at \\(dq/path/to/repo\\(dq\nEnter new passphrase:\nEnter same passphrase again:\nRemember your passphrase. Your data will be inaccessible without it.\nKey in \\(dq/root/.config/borg/keys/mnt_backup\\(dq created.\nKeep this key safe. Your data will be inaccessible without it.\nSynchronizing chunks cache...\nArchives: 0, w/ cached Idx: 0, w/ outdated Idx: 0, w/o cached Idx: 0.\nDone.\n\n# Change key file passphrase\n$ borg key change\\-passphrase \\-v\nEnter passphrase for key /root/.config/borg/keys/mnt_backup:\nEnter new passphrase:\nEnter same passphrase again:\nRemember your passphrase. Your data will be inaccessible without it.\nKey updated\n.EE\n.UNINDENT\n.UNINDENT\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\nThe key file paths shown above are the defaults for Linux (\\fB~/.config/borg/keys/\\fP).\nOn macOS, key files are stored in \\fB~/Library/Application Support/borg/keys/\\fP\\&.\nOn Windows, they are stored in \\fBC:\\eUsers\\e\\eAppData\\eRoaming\\eborg\\ekeys\\e\\fP\\&.\nSee \\fIenv_vars\\fP for details.\n.UNINDENT\n.UNINDENT\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Import a previously\\-exported key into the specified\n# key file (creating or overwriting the output key)\n# (keyfile repositories only)\n$ BORG_KEY_FILE=/path/to/output\\-key borg key import /path/to/exported\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nFully automated using environment variables:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ BORG_NEW_PASSPHRASE=old borg repo\\-create \\-\\-encryption=repokey\\-aes\\-ocb\n# now \\(dqold\\(dq is the current passphrase.\n$ BORG_PASSPHRASE=old BORG_NEW_PASSPHRASE=new borg key change\\-passphrase\n# now \\(dqnew\\(dq is the current passphrase.\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-key-export.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-key-export\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-key-export \\- Exports the repository key for backup.\n.SH SYNOPSIS\n.sp\nborg [common options] key export [options] [PATH]\n.SH DESCRIPTION\n.sp\nThis command backs up the borg key.\n.sp\nIf repository encryption is used, the repository is inaccessible\nwithout the borg key (and the passphrase that protects the borg key).\nIf a repository is not encrypted, but authenticated, the borg key is\nstill needed to access the repository normally.\n.sp\nFor repositories using \\fBkeyfile\\fP encryption the key is kept locally\non the system that is capable of doing backups. To guard against loss\nor corruption of this key, the key needs to be backed up independently\nof the main data backup.\n.sp\nFor repositories using \\fBrepokey\\fP encryption or \\fBauthenticated\\fP mode\nthe key is kept in the repository. A backup is thus not strictly needed,\nbut guards against the repository becoming inaccessible if the key is\ncorrupted or lost.\n.sp\nNote that the backup produced does not include the passphrase itself\n(i.e. the exported key stays encrypted). In order to regain access to a\nrepository, one needs both the exported key and the original passphrase.\nKeep the exported key and the passphrase at safe places.\n.sp\nThere are three backup formats. The normal backup format is suitable for\ndigital storage as a file. The \\fB\\-\\-paper\\fP backup format is optimized\nfor printing and typing in while importing, with per line checks to\nreduce problems with manual input. The \\fB\\-\\-qr\\-html\\fP creates a printable\nHTML template with a QR code and a copy of the \\fB\\-\\-paper\\fP\\-formatted key.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B PATH\nwhere to store the backup\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-paper\nCreate an export suitable for printing and later type\\-in\n.TP\n.B \\-\\-qr\\-html\nCreate an HTML file suitable for printing and later type\\-in or QR scan\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\nborg key export > encrypted\\-key\\-backup\nborg key export \\-\\-paper > encrypted\\-key\\-backup.txt\nborg key export \\-\\-qr\\-html > encrypted\\-key\\-backup.html\n# Or pass the output file as an argument instead of redirecting stdout:\nborg key export encrypted\\-key\\-backup\nborg key export \\-\\-paper encrypted\\-key\\-backup.txt\nborg key export \\-\\-qr\\-html encrypted\\-key\\-backup.html\n.EE\n.UNINDENT\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-key\\-import(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-key-import.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-key-import\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-key-import \\- Imports the repository key from backup.\n.SH SYNOPSIS\n.sp\nborg [common options] key import [options] [PATH]\n.SH DESCRIPTION\n.sp\nThis command restores a key previously backed up with the export command.\n.sp\nIf the \\fB\\-\\-paper\\fP option is given, the import will be an interactive\nprocess in which each line is checked for plausibility before\nproceeding to the next line. For this format PATH must not be given.\n.sp\nFor repositories using keyfile encryption, the key file which \\fBborg key\nimport\\fP writes to depends on several factors. If the \\fBBORG_KEY_FILE\\fP\nenvironment variable is set and non\\-empty, \\fBborg key import\\fP creates\nor overwrites that file named by \\fB$BORG_KEY_FILE\\fP\\&. Otherwise, \\fBborg\nkey import\\fP searches in the \\fB$BORG_KEYS_DIR\\fP directory for a key file\nassociated with the repository. If a key file is found in\n\\fB$BORG_KEYS_DIR\\fP, \\fBborg key import\\fP overwrites it; otherwise, \\fBborg\nkey import\\fP creates a new key file in \\fB$BORG_KEYS_DIR\\fP\\&.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B PATH\npath to the backup (\\(aq\\-\\(aq to read from stdin)\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-paper\ninteractively import from a backup done with \\fB\\-\\-paper\\fP\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-key\\-export(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-key-migrate-to-repokey.1", "content": ".\\\" Man page generated from reStructuredText.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"BORG-KEY-MIGRATE-TO-REPOKEY\" 1 \"2022-02-19\" \"\" \"borg backup tool\"\n.SH NAME\nborg-key-migrate-to-repokey \\- Migrate passphrase -> repokey\n.SH SYNOPSIS\n.sp\nborg [common options] key migrate\\-to\\-repokey [options] [REPOSITORY]\n.SH DESCRIPTION\n.sp\nThis command migrates a repository from passphrase mode (removed in Borg 1.0)\nto repokey mode.\n.sp\nYou will be first asked for the repository passphrase (to open it in passphrase\nmode). This is the same passphrase as you used to use for this repo before 1.0.\n.sp\nIt will then derive the different secrets from this passphrase.\n.sp\nThen you will be asked for a new passphrase (twice, for safety). This\npassphrase will be used to protect the repokey (which contains these same\nsecrets in encrypted form). You may use the same passphrase as you used to\nuse, but you may also use a different one.\n.sp\nAfter migrating to repokey mode, you can change the passphrase at any time.\nBut please note: the secrets will always stay the same and they could always\nbe derived from your (old) passphrase\\-mode passphrase.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.sp\nREPOSITORY\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP\n.SH AUTHOR\nThe Borg Collective\n.\\\" Generated by docutils manpage writer.\n.\n" }, { "path": "docs/man/borg-key.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-key\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-key \\- Manage the keyfile or repokey of a repository\n.SH SYNOPSIS\n.nf\nborg [common options] key export ...\nborg [common options] key import ...\nborg [common options] key change\\-passphrase ...\nborg [common options] key change\\-location ...\n.fi\n.sp\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-key\\-export(1)\\fP, \\fIborg\\-key\\-import(1)\\fP, \\fIborg\\-key\\-change\\-passphrase(1)\\fP, \\fIborg\\-key\\-change\\-location(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-list.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-list\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-list \\- List archive contents.\n.SH SYNOPSIS\n.sp\nborg [common options] list [options] NAME [PATH...]\n.SH DESCRIPTION\n.sp\nThis command lists the contents of an archive.\n.sp\nFor more help on include/exclude patterns, see the output of \\fIborg_patterns\\fP\\&.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B NAME\nspecify the archive name\n.TP\n.B PATH\npaths to list; patterns are supported\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-\\-short\nonly print file/directory names, nothing else\n.TP\n.BI \\-\\-format \\ FORMAT\nspecify format for file listing (default: \\(dq{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\\(dq)\n.TP\n.B \\-\\-json\\-lines\nFormat output as JSON Lines. The form of \\fB\\-\\-format\\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text.\n.TP\n.BI \\-\\-depth \\ N\nonly list files up to the specified directory depth\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.UNINDENT\n.SH EXAMPLES\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ borg list root\\-2016\\-02\\-15\ndrwxr\\-xr\\-x root root 0 Mon, 2016\\-02\\-15 17:44:27 .\ndrwxrwxr\\-x root root 0 Mon, 2016\\-02\\-15 19:04:49 bin\n\\-rwxr\\-xr\\-x root root 1029624 Thu, 2014\\-11\\-13 00:08:51 bin/bash\nlrwxrwxrwx root root 0 Fri, 2015\\-03\\-27 20:24:26 bin/bzcmp \\-> bzdiff\n\\-rwxr\\-xr\\-x root root 2140 Fri, 2015\\-03\\-27 20:24:22 bin/bzdiff\n\\&...\n\n$ borg list root\\-2016\\-02\\-15 \\-\\-pattern \\(dq\\- bin/ba*\\(dq\ndrwxr\\-xr\\-x root root 0 Mon, 2016\\-02\\-15 17:44:27 .\ndrwxrwxr\\-x root root 0 Mon, 2016\\-02\\-15 19:04:49 bin\nlrwxrwxrwx root root 0 Fri, 2015\\-03\\-27 20:24:26 bin/bzcmp \\-> bzdiff\n\\-rwxr\\-xr\\-x root root 2140 Fri, 2015\\-03\\-27 20:24:22 bin/bzdiff\n\\&...\n\n$ borg list archiveA \\-\\-format=\\(dq{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}\\(dq\ndrwxrwxr\\-x user user 0 Sun, 2015\\-02\\-01 11:00:00 .\ndrwxrwxr\\-x user user 0 Sun, 2015\\-02\\-01 11:00:00 code\ndrwxrwxr\\-x user user 0 Sun, 2015\\-02\\-01 11:00:00 code/myproject\n\\-rw\\-rw\\-r\\-\\- user user 1416192 Sun, 2015\\-02\\-01 11:00:00 code/myproject/file.ext\n\\-rw\\-rw\\-r\\-\\- user user 1416192 Sun, 2015\\-02\\-01 11:00:00 code/myproject/file.text\n\\&...\n\n$ borg list archiveA \\-\\-pattern \\(aq+ re:\\e.ext$\\(aq \\-\\-pattern \\(aq\\- re:^.*$\\(aq\n\\-rw\\-rw\\-r\\-\\- user user 1416192 Sun, 2015\\-02\\-01 11:00:00 code/myproject/file.ext\n\\&...\n\n$ borg list archiveA \\-\\-pattern \\(aq+ re:.ext$\\(aq \\-\\-pattern \\(aq\\- re:^.*$\\(aq\n\\-rw\\-rw\\-r\\-\\- user user 1416192 Sun, 2015\\-02\\-01 11:00:00 code/myproject/file.ext\n\\-rw\\-rw\\-r\\-\\- user user 1416192 Sun, 2015\\-02\\-01 11:00:00 code/myproject/file.text\n\\&...\n.EE\n.UNINDENT\n.UNINDENT\n.SH NOTES\n.SS The FORMAT specifier syntax\n.sp\nThe \\fB\\-\\-format\\fP option uses Python\\(aqs format string syntax \\%\\&.\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n$ borg list \\-\\-format \\(aq{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\\(aq ArchiveFoo\n\\-rw\\-rw\\-r\\-\\- user user 1024 Thu, 2021\\-12\\-09 10:22:17 file\\-foo\n\\&...\n\n# {VAR:NUMBER} \\- pad to NUMBER columns right\\-aligned.\n$ borg list \\-\\-format \\(aq{mode} {user:>6} {group:>6} {size:<8} {mtime} {path}{extra}{NL}\\(aq ArchiveFoo\n\\-rw\\-rw\\-r\\-\\- user user 1024 Thu, 2021\\-12\\-09 10:22:17 file\\-foo\n\\&...\n.EE\n.UNINDENT\n.UNINDENT\n.sp\nThe following keys are always available:\n\\- NEWLINE: OS dependent line separator\n\\- NL: alias of NEWLINE\n\\- NUL: NUL character for creating print0 / xargs \\-0 like output\n\\- SPACE: space character\n\\- TAB: tab character\n\\- CR: carriage return character\n\\- LF: line feed character\n.sp\nKeys available only when listing files in an archive:\n.INDENT 0.0\n.IP \\(bu 2\ntype: file type (file, dir, symlink, ...)\n.IP \\(bu 2\nmode: file mode (as in stat)\n.IP \\(bu 2\nuid: user id of file owner\n.IP \\(bu 2\ngid: group id of file owner\n.IP \\(bu 2\nuser: user name of file owner\n.IP \\(bu 2\ngroup: group name of file owner\n.IP \\(bu 2\npath: file path\n.IP \\(bu 2\ntarget: link target for symlinks\n.IP \\(bu 2\nhlid: hard link identity (same if hardlinking same fs object)\n.IP \\(bu 2\ninode: inode number\n.IP \\(bu 2\nflags: file flags\n.IP \\(bu 2\nsize: file size\n.IP \\(bu 2\nnum_chunks: number of chunks in this file\n.IP \\(bu 2\nmtime: file modification time\n.IP \\(bu 2\nctime: file change time\n.IP \\(bu 2\natime: file access time\n.IP \\(bu 2\nisomtime: file modification time (ISO 8601 format)\n.IP \\(bu 2\nisoctime: file change time (ISO 8601 format)\n.IP \\(bu 2\nisoatime: file access time (ISO 8601 format)\n.IP \\(bu 2\nfingerprint: Fingerprint of the file content (may have false negatives), format: H(conditions)\\-H(chunk_ids)\n.IP \\(bu 2\nblake2b\n.IP \\(bu 2\nblake2s\n.IP \\(bu 2\nmd5\n.IP \\(bu 2\nsha1\n.IP \\(bu 2\nsha224\n.IP \\(bu 2\nsha256\n.IP \\(bu 2\nsha384\n.IP \\(bu 2\nsha3_224\n.IP \\(bu 2\nsha3_256\n.IP \\(bu 2\nsha3_384\n.IP \\(bu 2\nsha3_512\n.IP \\(bu 2\nsha512\n.IP \\(bu 2\nxxh64: XXH64 checksum of this file (note: this is NOT a cryptographic hash!)\n.IP \\(bu 2\narchiveid: internal ID of the archive\n.IP \\(bu 2\narchivename: name of the archive\n.IP \\(bu 2\nextra: prepends {target} with \\(dq \\-> \\(dq for soft links and \\(dq link to \\(dq for hard links\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-info(1)\\fP, \\fIborg\\-diff(1)\\fP, \\fIborg\\-prune(1)\\fP, \\fIborg\\-patterns(1)\\fP, \\fIborg\\-repo\\-list(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-match-archives.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-match-archives\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-match-archives \\- Details regarding match-archives\n.SH DESCRIPTION\n.sp\nThe \\fB\\-\\-match\\-archives\\fP option matches a given pattern against the list of all archives\nin the repository. It can be given multiple times.\n.sp\nThe patterns can have a prefix of:\n.INDENT 0.0\n.IP \\(bu 2\nname: pattern match on the archive name (default)\n.IP \\(bu 2\naid: prefix match on the archive id (only one result allowed)\n.IP \\(bu 2\nuser: exact match on the username who created the archive\n.IP \\(bu 2\nhost: exact match on the hostname where the archive was created\n.IP \\(bu 2\ntags: match on the archive tags\n.UNINDENT\n.sp\nIn case of a name pattern match,\nit uses pattern styles similar to the ones described by \\fBborg help patterns\\fP:\n.INDENT 0.0\n.TP\n.B Identical match pattern, selector \\fBid:\\fP (default)\nSimple string match, must fully match exactly as given.\n.TP\n.B Shell\\-style patterns, selector \\fBsh:\\fP\nMatch like on the shell, wildcards like \\fI*\\fP and \\fI?\\fP work.\n.TP\n.B Regular expressions \\%, selector \\fBre:\\fP\nFull regular expression support.\nThis is very powerful, but can also get rather complicated.\n.UNINDENT\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# name match, id: style\nborg delete \\-\\-match\\-archives \\(aqid:archive\\-with\\-crap\\(aq\nborg delete \\-a \\(aqid:archive\\-with\\-crap\\(aq # same, using short option\nborg delete \\-a \\(aqarchive\\-with\\-crap\\(aq # same, because \\(aqid:\\(aq is the default\n\n# name match, sh: style\nborg delete \\-a \\(aqsh:home\\-kenny\\-*\\(aq\n\n# name match, re: style\nborg delete \\-a \\(aqre:pc[123]\\-home\\-(user1|user2)\\-2022\\-09\\-.*\\(aq\n\n# archive id prefix match:\nborg delete \\-a \\(aqaid:d34db33f\\(aq\n\n# host or user match\nborg delete \\-a \\(aquser:kenny\\(aq\nborg delete \\-a \\(aqhost:kenny\\-pc\\(aq\n\n# tags match\nborg delete \\-a \\(aqtags:TAG1\\(aq \\-a \\(aqtags:TAG2\\(aq\n.EE\n.UNINDENT\n.UNINDENT\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-mount.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-mount\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-mount \\- Mounts an archive or an entire repository as a FUSE filesystem.\n.SH SYNOPSIS\n.sp\nborg [common options] mount [options] MOUNTPOINT [PATH...]\n.SH DESCRIPTION\n.sp\nThis command mounts a repository or an archive as a FUSE filesystem.\nThis can be useful for browsing or restoring individual files.\n.sp\nWhen restoring, take into account that the current FUSE implementation does\nnot support special fs flags and ACLs.\n.sp\nWhen mounting a repository, the top directories will be named like the\narchives and the directory structure below these will be loaded on\\-demand from\nthe repository when entering these directories, so expect some delay.\n.sp\nCare should be taken, as Borg backs up symlinks as\\-is. When an archive\nor repository is mounted, it is possible to “jump” outside the mount point\nby following a symlink. If this happens, files or directories (or versions of them)\nthat are not part of the archive or repository may appear to be within the mount point.\n.sp\nUnless the \\fB\\-\\-foreground\\fP option is given, the command will run in the\nbackground until the filesystem is \\fBunmounted\\fP\\&.\n.sp\nPerformance tips:\n.INDENT 0.0\n.IP \\(bu 2\nWhen doing a \\(dqwhole repository\\(dq mount:\ndo not enter archive directories if not needed; this avoids on\\-demand loading.\n.IP \\(bu 2\nOnly mount a specific archive, not the whole repository.\n.IP \\(bu 2\nOnly mount specific paths in a specific archive, not the complete archive.\n.UNINDENT\n.sp\nThe command \\fBborgfs\\fP provides a wrapper for \\fBborg mount\\fP\\&. This can also be\nused in fstab entries:\n\\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0\\fP\n.sp\nTo allow a regular user to use fstab entries, add the \\fBuser\\fP option:\n\\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0\\fP\n.sp\nFor FUSE configuration and mount options, see the mount.fuse(8) manual page.\n.sp\nBorg\\(aqs default behavior is to use the archived user and group names of each\nfile and map them to the system\\(aqs respective user and group IDs.\nAlternatively, using \\fBnumeric\\-ids\\fP will instead use the archived user and\ngroup IDs without any mapping.\n.sp\nThe \\fBuid\\fP and \\fBgid\\fP mount options (implemented by Borg) can be used to\noverride the user and group IDs of all files (i.e., \\fBborg mount \\-o\nuid=1000,gid=1000\\fP).\n.sp\nThe man page references \\fBuser_id\\fP and \\fBgroup_id\\fP mount options\n(implemented by FUSE) which specify the user and group ID of the mount owner\n(also known as the user who does the mounting). It is set automatically by libfuse (or\nthe filesystem if libfuse is not used). However, you should not specify these\nmanually. Unlike the \\fBuid\\fP and \\fBgid\\fP mount options, which affect all files,\n\\fBuser_id\\fP and \\fBgroup_id\\fP affect the user and group ID of the mounted\n(base) directory.\n.sp\nAdditional mount options supported by Borg:\n.INDENT 0.0\n.IP \\(bu 2\n\\fBversions\\fP: when used with a repository mount, this gives a merged, versioned\nview of the files in the archives. EXPERIMENTAL; layout may change in the future.\n.IP \\(bu 2\n\\fBallow_damaged_files\\fP: by default, damaged files (where chunks are missing)\nwill return EIO (I/O error) when trying to read the related parts of the file.\nSet this option to replace the missing parts with all\\-zero bytes.\n.IP \\(bu 2\n\\fBignore_permissions\\fP: for security reasons the \\fBdefault_permissions\\fP mount\noption is internally enforced by Borg. \\fBignore_permissions\\fP can be given to\nnot enforce \\fBdefault_permissions\\fP\\&.\n.UNINDENT\n.sp\nThe BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is intended for advanced users\nto tweak performance. It sets the number of cached data chunks; additional\nmemory usage can be up to ~8 MiB times this number. The default is the number\nof CPU cores.\n.sp\nWhen the daemonized process receives a signal or crashes, it does not unmount.\nUnmounting in these cases could cause an active rsync or similar process\nto delete data unintentionally.\n.sp\nWhen running in the foreground, ^C/SIGINT cleanly unmounts the filesystem,\nbut other signals or crashes do not.\n.sp\nDebugging:\n.sp\n\\fBborg mount\\fP usually daemonizes and the daemon process sends stdout/stderr\nto /dev/null. Thus, you need to either use \\fB\\-f / \\-\\-foreground\\fP to make it stay\nin the foreground and not daemonize, or use \\fBBORG_LOGGING_CONF\\fP to reconfigure\nthe logger to output to a file.\n.SH OPTIONS\n.sp\nSee \\fIborg\\-common(1)\\fP for common options of Borg commands.\n.SS arguments\n.INDENT 0.0\n.TP\n.B MOUNTPOINT\nwhere to mount the filesystem\n.TP\n.B PATH\npaths to extract; patterns are supported\n.UNINDENT\n.SS options\n.INDENT 0.0\n.TP\n.B \\-f\\fP,\\fB \\-\\-foreground\nstay in foreground, do not daemonize\n.TP\n.B \\-o\nextra mount options\n.TP\n.B \\-\\-numeric\\-ids\nuse numeric user and group identifiers from archives\n.UNINDENT\n.SS Archive filters\n.INDENT 0.0\n.TP\n.BI \\-a \\ PATTERN\\fR,\\fB \\ \\-\\-match\\-archives \\ PATTERN\nonly consider archives matching all patterns. See \\(dqborg help match\\-archives\\(dq.\n.TP\n.BI \\-\\-sort\\-by \\ KEYS\nComma\\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp\n.TP\n.BI \\-\\-first \\ N\nconsider the first N archives after other filters are applied\n.TP\n.BI \\-\\-last \\ N\nconsider the last N archives after other filters are applied\n.TP\n.BI \\-\\-oldest \\ TIMESPAN\nconsider archives between the oldest archive\\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newest \\ TIMESPAN\nconsider archives between the newest archive\\(aqs timestamp and (newest \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-older \\ TIMESPAN\nconsider archives older than (now \\- TIMESPAN), e.g., 7d or 12m.\n.TP\n.BI \\-\\-newer \\ TIMESPAN\nconsider archives newer than (now \\- TIMESPAN), e.g., 7d or 12m.\n.UNINDENT\n.SS Include/Exclude options\n.INDENT 0.0\n.TP\n.BI \\-e \\ PATTERN\\fR,\\fB \\ \\-\\-exclude \\ PATTERN\nexclude paths matching PATTERN\n.TP\n.BI \\-\\-exclude\\-from \\ EXCLUDEFILE\nread exclude patterns from EXCLUDEFILE, one per line\n.TP\n.BI \\-\\-pattern \\ PATTERN\ninclude/exclude paths matching PATTERN\n.TP\n.BI \\-\\-patterns\\-from \\ PATTERNFILE\nread include/exclude patterns from PATTERNFILE, one per line\n.TP\n.BI \\-\\-strip\\-components \\ NUMBER\nRemove the specified number of leading path elements. Paths with fewer elements will be silently skipped.\n.UNINDENT\n.SH SEE ALSO\n.sp\n\\fIborg\\-common(1)\\fP, \\fIborg\\-umount(1)\\fP, \\fIborg\\-extract(1)\\fP\n.SH Author\nThe Borg Collective\n.\\\" End of generated man page.\n" }, { "path": "docs/man/borg-patterns.1", "content": ".\\\" Man page generated from reStructuredText\n.\\\" by the Docutils 0.22.4 manpage writer.\n.\n.\n.nr rst2man-indent-level 0\n.\n.de1 rstReportMargin\n\\\\$1 \\\\n[an-margin]\nlevel \\\\n[rst2man-indent-level]\nlevel margin: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n-\n\\\\n[rst2man-indent0]\n\\\\n[rst2man-indent1]\n\\\\n[rst2man-indent2]\n..\n.de1 INDENT\n.\\\" .rstReportMargin pre:\n. RS \\\\$1\n. nr rst2man-indent\\\\n[rst2man-indent-level] \\\\n[an-margin]\n. nr rst2man-indent-level +1\n.\\\" .rstReportMargin post:\n..\n.de UNINDENT\n. RE\n.\\\" indent \\\\n[an-margin]\n.\\\" old: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.nr rst2man-indent-level -1\n.\\\" new: \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]\n.in \\\\n[rst2man-indent\\\\n[rst2man-indent-level]]u\n..\n.TH \"borg-patterns\" \"1\" \"2026-03-15\" \"\" \"borg backup tool\"\n.SH Name\nborg-patterns \\- Details regarding patterns\n.SH DESCRIPTION\n.sp\nWhen specifying one or more file paths in a Borg command that supports\npatterns for the respective option or argument, you can apply the\npatterns described here to include only desired files and/or exclude\nunwanted ones. Patterns can be used\n.INDENT 0.0\n.IP \\(bu 2\nfor \\fB\\-\\-exclude\\fP option,\n.IP \\(bu 2\nin the file given with \\fB\\-\\-exclude\\-from\\fP option,\n.IP \\(bu 2\nfor \\fB\\-\\-pattern\\fP option,\n.IP \\(bu 2\nin the file given with \\fB\\-\\-patterns\\-from\\fP option and\n.IP \\(bu 2\nfor \\fBPATH\\fP arguments that explicitly support them.\n.UNINDENT\n.sp\nThe path/filenames used as input for the pattern matching start with the\ncurrently active recursion root. You usually give the recursion root(s)\nwhen invoking borg and these can be either relative or absolute paths.\n.sp\nBe careful, your patterns must match the archived paths:\n.INDENT 0.0\n.IP \\(bu 2\nArchived paths never start with a leading slash (\\(aq/\\(aq), nor with \\(aq.\\(aq, nor with \\(aq..\\(aq.\n.INDENT 2.0\n.IP \\(bu 2\nWhen you back up absolute paths like \\fB/home/user\\fP, the archived\npaths start with \\fBhome/user\\fP\\&.\n.IP \\(bu 2\nWhen you back up relative paths like \\fB\\&./src\\fP, the archived paths\nstart with \\fBsrc\\fP\\&.\n.IP \\(bu 2\nWhen you back up relative paths like \\fB\\&../../src\\fP, the archived paths\nstart with \\fBsrc\\fP\\&.\n.IP \\(bu 2\nOn native Windows, archived absolute paths look like \\fBC/Windows/System32\\fP\\&.\n.UNINDENT\n.UNINDENT\n.sp\nBorg supports different pattern styles. To define a non\\-default\nstyle for a specific pattern, prefix it with two characters followed\nby a colon \\(aq:\\(aq (i.e. \\fBfm:path/*\\fP, \\fBsh:path/**\\fP).\n.sp\nNote: Windows users must only use forward slashes in patterns, not backslashes.\n.sp\nThe default pattern style for \\fB\\-\\-exclude\\fP differs from \\fB\\-\\-pattern\\fP, see below.\n.INDENT 0.0\n.TP\n.B Fnmatch \\%, selector \\fBfm:\\fP\nThis is the default style for \\fB\\-\\-exclude\\fP and \\fB\\-\\-exclude\\-from\\fP\\&.\nThese patterns use a variant of shell pattern syntax, with \\(aq*\\(aq matching\nany number of characters, \\(aq?\\(aq matching any single character, \\(aq[...]\\(aq\nmatching any single character specified, including ranges, and \\(aq[!...]\\(aq\nmatching any character not specified. For the purpose of these patterns,\nthe path separator (forward slash \\(aq/\\(aq) is not treated specially.\nWrap meta\\-characters in brackets for a literal\nmatch (i.e. \\fB[?]\\fP to match the literal character \\(aq?\\(aq). For a path\nto match a pattern, the full path must match, or it must match\nfrom the start of the full path to just before a path separator. Except\nfor the root path, paths will never end in the path separator when\nmatching is attempted. Thus, if a given pattern ends in a path\nseparator, a \\(aq*\\(aq is appended before matching is attempted. A leading\npath separator is always removed.\n.TP\n.B Shell\\-style patterns, selector \\fBsh:\\fP\nThis is the default style for \\fB\\-\\-pattern\\fP and \\fB\\-\\-patterns\\-from\\fP\\&.\nLike fnmatch patterns these are similar to shell patterns. The difference\nis that the pattern may include \\fB**/\\fP for matching zero or more directory\nlevels, \\fB*\\fP for matching zero or more arbitrary characters with the\nexception of any path separator, \\fB{}\\fP containing comma\\-separated\nalternative patterns. A leading path separator is always removed.\n.TP\n.B Regular expressions \\%, selector \\fBre:\\fP\nUnlike shell patterns, regular expressions are not required to match the full\npath and any substring match is sufficient. It is strongly recommended to\nanchor patterns to the start (\\(aq^\\(aq), to the end (\\(aq$\\(aq) or both.\n.TP\n.B Path prefix, selector \\fBpp:\\fP\nThis pattern style is useful to match whole subdirectories. The pattern\n\\fBpp:root/somedir\\fP matches \\fBroot/somedir\\fP and everything therein.\nA leading path separator is always removed.\n.TP\n.B Path full\\-match, selector \\fBpf:\\fP\nThis pattern style is (only) useful to match full paths.\nThis is kind of a pseudo pattern as it cannot have any variable or\nunspecified parts \\- the full path must be given. \\fBpf:root/file.ext\\fP\nmatches \\fBroot/file.ext\\fP only. A leading path separator is always\nremoved.\n.sp\nImplementation note: this is implemented via very time\\-efficient O(1)\nhashtable lookups (this means you can have huge amounts of such patterns\nwithout impacting performance much).\nDue to that, this kind of pattern does not respect any context or order.\nIf you use such a pattern to include a file, it will always be included\n(if the directory recursion encounters it).\nOther include/exclude patterns that would normally match will be ignored.\nSame logic applies for exclude.\n.UNINDENT\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\n\\fBre:\\fP, \\fBsh:\\fP and \\fBfm:\\fP patterns are all implemented on top of\nthe Python SRE engine. It is very easy to formulate patterns for each\nof these types which requires an inordinate amount of time to match\npaths. If untrusted users are able to supply patterns, ensure they\ncannot supply \\fBre:\\fP patterns. Further, ensure that \\fBsh:\\fP and\n\\fBfm:\\fP patterns only contain a handful of wildcards at most.\n.UNINDENT\n.UNINDENT\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\n\\fBWindows path handling\\fP: All paths in Borg archives use forward slashes (\\fB/\\fP)\nas path separators, regardless of the platform. When creating archives on Windows,\nbackslashes from filesystem paths are automatically converted to forward slashes.\n.UNINDENT\n.UNINDENT\n.sp\n\\fBNote:\\fP\n.INDENT 0.0\n.INDENT 3.5\n\\fBWindows reserved characters\\fP: On Windows, when extracting archives created on\nPOSIX systems, paths may contain characters that are reserved from being used in\nfile or directory names (like: \\fB< > : \\(dq \\e | ? *\\fP).\nThese are replaced by characters in the unicode private use area (\\fBU+F0xx\\fP) like\nthe CIFS mapchars feature also does it. It won\\(aqt be pretty, but at least it works.\n.UNINDENT\n.UNINDENT\n.sp\nExclusions can be passed via the command line option \\fB\\-\\-exclude\\fP\\&. When used\nfrom within a shell, the patterns should be quoted to protect them from\nexpansion.\n.sp\nPatterns matching special characters, e.g. whitespace, within a shell may\nrequire adjustments, such as putting quotation marks around the arguments.\nExample:\nUsing bash, the following command line option would match and exclude \\(dqitem name\\(dq:\n\\fB\\-\\-pattern=\\(aq\\-path/item name\\(aq\\fP\nNote that when patterns are used within a pattern file directly read by borg,\ne.g. when using \\fB\\-\\-exclude\\-from\\fP or \\fB\\-\\-patterns\\-from\\fP, there is no shell\ninvolved and thus no quotation marks are required.\n.sp\nThe \\fB\\-\\-exclude\\-from\\fP option permits loading exclusion patterns from a text\nfile with one pattern per line. Lines empty or starting with the hash sign\n\\(aq#\\(aq after removing whitespace on both ends are ignored. The optional style\nselector prefix is also supported for patterns loaded from a file. Due to\nwhitespace removal, paths with whitespace at the beginning or end can only be\nexcluded using regular expressions.\n.sp\nTo test your exclusion patterns without performing an actual backup you can\nrun \\fBborg create \\-\\-list \\-\\-dry\\-run ...\\fP\\&.\n.sp\nExamples:\n.INDENT 0.0\n.INDENT 3.5\n.sp\n.EX\n# Exclude a directory anywhere in the tree named \\(ga\\(gasteamapps/common\\(ga\\(ga\n# (and everything below it), regardless of where it appears:\n$ borg create \\-e \\(aqsh:**/steamapps/common/**\\(aq archive /\n\n# Exclude the contents of \\(ga\\(ga/home/user/.cache\\(ga\\(ga:\n$ borg create \\-e \\(aqsh:home/user/.cache/**\\(aq archive /home/user\n$ borg create \\-e home/user/.cache/ archive /home/user\n\n# The file \\(aq/home/user/.cache/important\\(aq is *not* backed up:\n$ borg create \\-e home/user/.cache/ archive / /home/user/.cache/important\n\n# Exclude \\(aq/home/user/file.o\\(aq but not \\(aq/home/user/file.odt\\(aq:\n$ borg create \\-e \\(aq*.o\\(aq archive /\n\n# Exclude \\(aq/home/user/junk\\(aq and \\(aq/home/user/subdir/junk\\(aq but\n# not \\(aq/home/user/importantjunk\\(aq or \\(aq/etc/junk\\(aq:\n$ borg create \\-e \\(aqhome/*/junk\\(aq archive /\n\n# The contents of directories in \\(aq/home\\(aq are not backed up when their name\n# ends in \\(aq.tmp\\(aq\n$ borg create \\-\\-exclude \\(aqre:^home/[^/]+\\e.tmp/\\(aq archive /\n\n# Load exclusions from file\n$ cat >exclude.txt <