Repository: longhorn/longhorn Branch: master Commit: a2502a52c86f Files: 310 Total size: 2.8 MB Directory structure: gitextract_q7gmdmv5/ ├── .codespellignore ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug.yaml │ │ ├── ci.md │ │ ├── config.yml │ │ ├── doc.md │ │ ├── epic.yaml │ │ ├── feature.yaml │ │ ├── fix-cve-issues-for-release.md │ │ ├── hotfix.md │ │ ├── improvement.yaml │ │ ├── infra.md │ │ ├── performance-benchmark.md │ │ ├── refactor.md │ │ ├── regular-tasks-for-feature-release.md │ │ ├── release.md │ │ ├── task.md │ │ └── test.md │ ├── PULL_REQUEST_TEMPLATE.md │ ├── mergify.yml │ └── workflows/ │ ├── add-issue-to-projects.yml │ ├── auto-author-assign.yml │ ├── backport-pr.yml │ ├── check-sprint-last-day.py │ ├── close-issue.yml │ ├── codespell.yml │ ├── conventional-commits.yml │ ├── create-issue.yml │ ├── create-release-task-issues.yml │ ├── periodic-issue-sprint-update.yml │ ├── pr-review-reminder.py │ ├── pr-review-reminder.yml │ ├── scan-and-notify-testing-items.py │ ├── scan-and-notify-testing-items.yml │ ├── scorecards.yml │ ├── stale.yaml │ ├── update-branch-image-tags.yaml │ ├── update-community-issue.yml │ ├── update-longhorn-issue.yml │ ├── validate-yamls.yaml │ └── wont-fix.yml ├── .gitignore ├── ADOPTERS.md ├── CHANGELOG/ │ ├── CHANGELOG-1.10.0.md │ ├── CHANGELOG-1.10.1.md │ ├── CHANGELOG-1.10.2.md │ ├── CHANGELOG-1.11.0.md │ ├── CHANGELOG-1.11.1.md │ ├── CHANGELOG-1.4.0.md │ ├── CHANGELOG-1.4.1.md │ ├── CHANGELOG-1.4.2.md │ ├── CHANGELOG-1.4.3.md │ ├── CHANGELOG-1.5.0.md │ ├── CHANGELOG-1.5.1.md │ ├── CHANGELOG-1.5.2.md │ ├── CHANGELOG-1.5.3.md │ ├── CHANGELOG-1.5.4.md │ ├── CHANGELOG-1.5.5.md │ ├── CHANGELOG-1.6.0.md │ ├── CHANGELOG-1.6.1.md │ ├── CHANGELOG-1.6.2.md │ ├── CHANGELOG-1.6.3.md │ ├── CHANGELOG-1.6.4.md │ ├── CHANGELOG-1.7.0.md │ ├── CHANGELOG-1.7.1.md │ ├── CHANGELOG-1.7.2.md │ ├── CHANGELOG-1.7.3.md │ ├── CHANGELOG-1.8.0.md │ ├── CHANGELOG-1.8.1.md │ ├── CHANGELOG-1.8.2.md │ ├── CHANGELOG-1.9.0.md │ ├── CHANGELOG-1.9.1.md │ └── CHANGELOG-1.9.2.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MAINTAINERS ├── README.md ├── SECURITY.md ├── chart/ │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── README.md.gotmpl │ ├── app-readme.md │ ├── ocp-readme.md │ ├── questions.yaml │ ├── templates/ │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── clusterrole.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── crds.yaml │ │ ├── daemonset-sa.yaml │ │ ├── default-resource.yaml │ │ ├── default-setting.yaml │ │ ├── deployment-driver.yaml │ │ ├── deployment-ui.yaml │ │ ├── extra-objects.yaml │ │ ├── httproute.yaml │ │ ├── ingress.yaml │ │ ├── network-policies/ │ │ │ ├── backing-image-data-source-network-policy.yaml │ │ │ ├── backing-image-manager-network-policy.yaml │ │ │ ├── instance-manager-networking.yaml │ │ │ ├── manager-network-policy.yaml │ │ │ ├── recovery-backend-network-policy.yaml │ │ │ ├── ui-frontend-network-policy.yaml │ │ │ └── webhook-network-policy.yaml │ │ ├── postupgrade-job.yaml │ │ ├── preupgrade-job.yaml │ │ ├── priorityclass.yaml │ │ ├── psp.yaml │ │ ├── registry-secret.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ ├── serviceaccount.yaml │ │ ├── servicemonitor.yaml │ │ ├── services.yaml │ │ ├── storageclass.yaml │ │ ├── tls-secrets.yaml │ │ ├── uninstall-job.yaml │ │ └── validate-psp-install.yaml │ └── values.yaml ├── deploy/ │ ├── backupstores/ │ │ ├── README.md │ │ └── base/ │ │ ├── azurite/ │ │ │ ├── azurite-backupstore.yaml │ │ │ └── kustomization.yaml │ │ ├── cifs/ │ │ │ ├── cifs-backupstore.yaml │ │ │ └── kustomization.yaml │ │ ├── minio/ │ │ │ ├── kustomization.yaml │ │ │ └── minio-backupstore.yaml │ │ └── nfs/ │ │ ├── kustomization.yaml │ │ └── nfs-backupstore.yaml │ ├── longhorn-images.txt │ ├── longhorn-okd.yaml │ ├── longhorn.yaml │ ├── podsecuritypolicy.yaml │ ├── prerequisite/ │ │ ├── longhorn-cifs-installation.yaml │ │ ├── longhorn-gke-cos-node-agent.yaml │ │ └── longhorn-iscsi-selinux-workaround.yaml │ └── upgrade_responder_server/ │ ├── README.md │ ├── chart-values.yaml │ └── chart.yaml ├── dev/ │ ├── scale-test/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── sample.sh │ │ ├── scale-test.py │ │ └── statefulset.yaml │ ├── scripts/ │ │ ├── lm-update.sh │ │ └── update-image-pull-policy.sh │ └── upgrade-responder/ │ ├── README.md │ ├── install.sh │ └── manifests/ │ ├── grafana.yaml │ └── influxdb.yaml ├── enhancements/ │ ├── 20200319-default-disks-and-node-configuration.md │ ├── 20200331-replace-filesystem-id-key-in-disk-map.md │ ├── 20200625-volume-deletion-flows.md │ ├── 20200701-backupstore-file-locks.md │ ├── 20200721-refactor-restore-for-rebuild-enabling.md │ ├── 20200727-add-replica-eviction-support-for-disks-and-nodes.md │ ├── 20200817-improve-node-failure-handling.md │ ├── 20200819-keep-a-local-replica-to-engine.md │ ├── 20200821-add-revision-counter-disable-support.md │ ├── 20200821-rebuild-replica-with-existing-data.md │ ├── 20200904-csi-snapshot-support.md │ ├── 20200909-prometheus-support.md │ ├── 20201002-allow-recurring-backup-detached-volumes.md │ ├── 20201020-recover-from-volume-failure-by-delete-and-recreate-the-workload-pod.md │ ├── 20201106-disk-reconnection.md │ ├── 20201220-rwx-volume-support.md │ ├── 20210111-upgrade-engine-automatically.md │ ├── 20210125-enhanced-cpu-reservation.md │ ├── 20210216-volume-live-migration.md │ ├── 20210510-automatic-rebalance-replica.md │ ├── 20210525-async-pull-backups.md │ ├── 20210624-label-driven-recurring-job.md │ ├── 20210701-backing-image.md │ ├── 20210810-volume-clone.md │ ├── 20220110-extend-csi-snapshot-to-support-longhorn-snapshot.md │ ├── 20220317-snapshot-prune.md │ ├── 20220324-orphaned-data-cleanup.md │ ├── 20220408-support-kubernetes-ca.md │ ├── 20220420-longhorn-snapshot-crd.md │ ├── 20220428-storage-network-through-grpc-proxy.md │ ├── 20220727-dedicated-recovery-backend-for-rwx-volume-nfs-server.md │ ├── 20220801-failed-backups-cleanup.md │ ├── 20220913-longhorn-system-backup-restore.md │ ├── 20220922-snapshot-checksum-and-bit-rot-detection.md │ ├── 20221018-record-recurring-jobs-in-the-backup-volume.md │ ├── 20221024-longhorn-volumeattachment.md │ ├── 20221024-pv-encryption.md │ ├── 20221103-filesystem-trim.md │ ├── 20221109-support-bundle-enhancement.md │ ├── 20221123-local-volume.md │ ├── 20221205-concurrent-backup-restore-limit.md │ ├── 20221213-reimplement-longhorn-engine-with-SPDK.md │ ├── 20230103-recurring-snapshot-cleanup.md │ ├── 20230108-improve-backup-and-restore-efficiency-using-multiple-threads-and-compression-methods.md │ ├── 20230116-smb-cifs-backup-store-support.md │ ├── 20230303-consolidate-instance-managers.md │ ├── 20230307-pdb-for-longhon-csi-and-webhook.md │ ├── 20230309-recurring-filesystem-trim.md │ ├── 20230315-upgrade-path-enforcement.md │ ├── 20230417-extend-csi-snapshot-to-support-backingimage.md │ ├── 20230418-azure-blob-storage-backup-store-support.md │ ├── 20230420-engine-identity-validation.md │ ├── 20230420-upgrade-checker-info-collection.md │ ├── 20230517-set-recurring-job-to-pvc.md │ ├── 20230523-support-spdk-volumes.md │ ├── 20230526-volume-backup-policy-for-longhorn-system-backup.md │ ├── 20230601-forcibly-activate-a-restoring-dr-volume.md │ ├── 20230616-automatic-offline-replica-rebuild.md │ ├── 20230619-spdk-engine.md │ ├── 20230718-disk-anti-affinity.md │ ├── 20230807-backingimage-backup-support.md │ ├── 20230809-support-backup-and-restore-for-volumes-with-v2-data-engine.md │ ├── 20230814-talos-linux-support.md │ ├── 20230815-engine-upgrade-enforcement.md │ ├── 20230821-customize-maximum-recurring-job-retain-number.md │ ├── 20230905-automatically-evict-replicas-while-draining.md │ ├── 20230905-snapshot-space-management.md │ ├── 20231204-default-priority-class.md │ ├── 20231226-support-non-disruptive-volume-related-setting-updates.md │ ├── 20240108-replica-auto-balance-pressured-disks.md │ ├── 20240314-recurring-and-manual-full-backup-support.md │ ├── 20240325-nfs-server-ha.md │ ├── 20240402-share-manager-scheduling.md │ ├── 20240423-longhorn-commandline-interface.md │ ├── 20240426-backing-image-enhancement.md │ ├── 20240516-pre-pull-images.md │ ├── 20240522-storage-network-for-rwx-volumes.md │ ├── 20240612-backing-image-encryption-and-clone-support.md │ ├── 20240801-delete-backup-in-the-backupstore-asynchronously.md │ ├── 20240926-multiple-backup-targets-support.md │ ├── 20241003-improve-pulling-backups-from-the-backup-target.md │ ├── 20241028-auto-salvage-support-for-v2-volumes.md │ ├── 20241111-rwx-resize.md │ ├── 20241203-v2-backing-image-support.md │ ├── 20241206-v2-volume-live-migration.md │ ├── 20250107-v2-volume-encryption.md │ ├── 20250313-ublk-frontend-for-v2-engine.md │ ├── 20250331-orphaned-runtime-cleanup.md │ ├── 20250407-volume-offline-rebuilding.md │ ├── 20250508-v2-volume-cloning.md │ ├── 20250701-configurable-backup-block-size.md │ ├── 20250721-v2-engine-interrupt-mode.md │ ├── 20250721-v2-volume-expansion.md │ ├── 20250721-volume-offline-rebuilding-with-resource-awareness.md │ ├── 20251001-replica-balance-scheduling.md │ ├── 20251017-rwx-volume-endpoint-network.md │ ├── 20251119-snapshot-heavy-task-concurrency-control.md │ └── YYYYMMDD-template.md ├── examples/ │ ├── block/ │ │ ├── block_volume.yaml │ │ └── crypto/ │ │ ├── deployment_with_pvc.yaml │ │ ├── secret-crypto-global.yaml │ │ └── storageclass-crypto-global.yaml │ ├── crypto/ │ │ ├── secret-crypto-customized-rhel-FIPS-enabled.yaml │ │ ├── secret-crypto-customized.yaml │ │ ├── secret-crypto-global.yaml │ │ ├── storageclass-crypto-global.yaml │ │ ├── storageclass-crypto-per-volume-dedicated-namespace.yaml │ │ └── storageclass-crypto-per-volume.yaml │ ├── csi/ │ │ └── example_pv.yaml │ ├── data_migration.yaml │ ├── deployment.yaml │ ├── network-policy/ │ │ ├── backing-image-data-source-network-policy.yaml │ │ ├── backing-image-manager-network-policy.yaml │ │ ├── instance-manager-networking.yaml │ │ ├── manager-network-policy.yaml │ │ ├── recovery-backend-network-policy.yaml │ │ ├── ui-network-policy.yaml │ │ └── webhook-network-policy.yaml │ ├── pod_with_gev.yaml │ ├── pod_with_pvc.yaml │ ├── restore_to_file.yaml.template │ ├── rwx/ │ │ ├── rwx-nginx-deployment.yaml │ │ └── storageclass-migratable.yaml │ ├── simple_pod.yaml │ ├── simple_pvc.yaml │ ├── snapshot/ │ │ ├── existing_backup.yaml │ │ ├── restore_existing_backup.yaml │ │ ├── restore_pvc_snapshot.yaml │ │ ├── snapshot_existing.yaml │ │ ├── snapshot_pvc.yaml │ │ └── snapshotclass.yaml │ ├── statefulset.yaml │ ├── storageclass.yaml │ └── v2/ │ ├── pod_with_pvc.yaml │ └── storageclass.yaml ├── renovate.json ├── scalability/ │ ├── dev/ │ │ ├── control_plane_grafana_dashboard.json │ │ └── data-plane-grafana_dashboard.json │ └── reference-setup-performance-scalability-and-sizing-guidelines/ │ ├── README.md │ ├── on-prem/ │ │ ├── big-node-spec.md │ │ └── medium-node-spec.md │ └── public-cloud/ │ ├── big-node-spec.md │ └── medium-node-spec.md ├── scripts/ │ ├── generate-backupstore-credentials.sh │ ├── generate-longhorn-yaml.sh │ ├── helm-docs.sh │ ├── lhexec │ ├── load-images.sh │ ├── longhorn_rancher_chart_migration.sh │ ├── migrate-for-pre-070-volumes.sh │ ├── restore-backup-to-file.sh │ ├── save-images.sh │ ├── update-chart-questions.sh │ ├── update-chart-readme.sh │ ├── update-chart-values.sh │ ├── update-manifests-dev-version.sh │ └── update-uninstall-manifest.py ├── support-versions.txt └── uninstall/ └── uninstall.yaml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .codespellignore ================================================ aks ec2 eks gce gcp ================================================ FILE: .github/CODEOWNERS ================================================ * @longhorn/dev ================================================ FILE: .github/ISSUE_TEMPLATE/bug.yaml ================================================ name: Bug report description: Create a bug report title: "[BUG] " type: "Bug" labels: ["kind/bug", "require/qa-review-coverage", "require/backport"] assignees: - body: - type: markdown attributes: value: | Thanks for stopping by to let us know something could be better! For general questions or discussions, please use the [Discussions](https://github.com/longhorn/longhorn/discussions) tab. - type: textarea attributes: label: Describe the Bug description: A clear and concise description of the bug. validations: required: true - type: textarea attributes: label: To Reproduce description: Please provide the steps to reproduce the case. validations: required: false - type: textarea attributes: label: Expected Behavior description: A clear and concise description of what you expected to happen. validations: required: true - type: textarea attributes: label: Support Bundle for Troubleshooting description: Please provide a support bundle when the issue happens. You can generate a support bundle using the link at the footer of the Longhorn UI. Check [here](https://longhorn.io/docs/latest/troubleshoot/support-bundle/). Then, attach to the issue or send to longhorn-support-bundle@suse.com. validations: required: true - type: textarea attributes: label: Environment description: "Suggest checking the doc of the best practices of using Longhorn. [here](https://longhorn.io/docs/latest/best-practices)" value: | - Longhorn version: - Impacted volume (PV): - Installation method (e.g. Rancher Catalog App/Helm/Kubectl): - Kubernetes distro (e.g. RKE/K3s/EKS/OpenShift) and version: - Number of control plane nodes in the cluster: - Number of worker nodes in the cluster: - Node config - OS type and version: - Kernel version: - CPU per node: - Memory per node: - Disk type (e.g. SSD/NVMe/HDD): - Network bandwidth between the nodes (Gbps): - Underlying Infrastructure (e.g. on AWS/GCE, EKS/GKE, VMWare/KVM, Baremetal): - Number of Longhorn volumes in the cluster: validations: required: true - type: textarea attributes: label: Additional context description: Please add any other context about the problem here. validations: required: false - type: textarea attributes: label: Workaround and Mitigation description: Please add any workaround or mitigation to the problem here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/ci.md ================================================ --- name: CI task about: Create a CI task title: "[CI] " type: "Task" labels: ["kind/task", "area/ci"] assignees: '' --- ## What's the CI task to develop? Please describe ## Describe the items of the task development (DoD, definition of done) you'd like ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: GitHub Discussions url: https://github.com/longhorn/longhorn/discussions about: For getting answers to usage on GitHub, Discussions is even better than this bug tracker :) ================================================ FILE: .github/ISSUE_TEMPLATE/doc.md ================================================ --- name: Document about: Create or update document title: "[DOC] " type: "Doc" labels: kind/doc assignees: '' --- ## What's the document you plan to update? Why? Please describe ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/epic.yaml ================================================ name: Epic request description: Suggest an Epic for grouped features, enhancements or tasks title: "[EPIC] " type: "Epic" labels: [ "Epic" ] assignees: - body: - type: textarea attributes: label: Is your Epic request related to a problem? Please describe (👍 if you like this request) description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]. validations: required: true - type: textarea attributes: label: Describe the tasks for this Epic description: A clear and concise description of what you want to happen. For each sub-task, please create the corresponding issues and connect them to this Epic. validations: required: true - type: textarea attributes: label: Additional context placeholder: Add any other context or screenshots about the feature request here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/feature.yaml ================================================ name: Feature request description: Suggest an idea/feature title: "[FEATURE] " type: "Feature" labels: ["kind/feature", "require/lep", "require/doc", "require/auto-e2e-test", "require/manual-test-plan"] assignees: - body: - type: markdown attributes: value: | Thanks for stopping by to let us know something could be better! For general questions or discussions, please use the [Discussions](https://github.com/longhorn/longhorn/discussions) tab. - type: textarea attributes: label: Is your feature request related to a problem? Please describe (👍 if you like this request) description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]. validations: required: true - type: textarea attributes: label: Describe the solution you'd like description: A clear and concise description of what you want to happen. validations: required: false - type: textarea attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. validations: required: false - type: textarea attributes: label: Additional context placeholder: Add any other context or screenshots about the feature request here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/fix-cve-issues-for-release.md ================================================ --- name: Fix CVE Issues for Release about: Security issue fix task for a release title: "[RELEASE] Fix CVE Issues for {{ env.RELEASE_VERSION }}" type: "Task" labels: ["release/task", "area/security", "area/install-uninstall-upgrade"] assignees: '' --- ## What's the task? Please describe Identify and resolve CVE issues of Longhorn components: - https://github.com/longhorn/longhorn/blob/{{ env.BRANCH_NAME }}/deploy/longhorn-images.txt ## Describe the sub-tasks - [ ] Identify CVE issues in Longhorn components for {{ env.RELEASE_VERSION }}. - [ ] Resolve the identified CVE issues. - [ ] Perform CVE scans and possible fixes for each stage: Pre-RCs, RC1, RC2, etc. ## Additional context Keep this issue open until the version is released to consolidate all CVE-related fixes. This way, we can avoid creating additional CVE-related issues if new ones are identified later. ================================================ FILE: .github/ISSUE_TEMPLATE/hotfix.md ================================================ --- name: Hotfix about: Create a hotfix task title: "[HOTFIX] " type: "Task" labels: ["kind/hotfix", "require/important-note"] assignees: '' --- ## What's the task? Please describe ## Describe the sub-tasks - [ ] Create temporary branch for hotfixed component repo and push the fix to this branch - [ ] Validate the fix using hotfixed image - [ ] Create a tag for the hotfixed component repo. The naming convention is `vX.Y.Z-hotfix-`, e.g. `v1.4.0-hotfix-1`, `v1.4.0-hotfix-2`, etc. - [ ] Validate the fix using hotfixed image :`vX.Y.Z-hotfix-` - [ ] Remove the temporary branch - [ ] Update the important note in the official document vX.Y.Z ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/improvement.yaml ================================================ name: Improvement request description: Suggest an improvement of an existing feature title: "[IMPROVEMENT] " type: "Improvement" labels: ["kind/improvement", "require/doc", "require/manual-test-plan", "require/backport"] assignees: - body: - type: markdown attributes: value: | Thanks for stopping by to let us know something could be better! For general questions or discussions, please use the [Discussions](https://github.com/longhorn/longhorn/discussions) tab. - type: textarea attributes: label: Is your improvement request related to a feature? Please describe (👍 if you like this request) description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]. validations: required: true - type: textarea attributes: label: Describe the solution you'd like description: A clear and concise description of what you want to happen. validations: required: false - type: textarea attributes: label: Describe alternatives you've considered placeholder: A clear and concise description of any alternative solutions or features you've considered. validations: required: false - type: textarea attributes: label: Additional context description: Add any other context or screenshots about the feature request here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/infra.md ================================================ --- name: Infra about: Create a test/dev infra task title: "[INFRA] " type: "Task" labels: ["kind/task", "area/infra"] assignees: '' --- ## What's the infra task? Please describe ## Describe the items of the infra task (DoD, definition of done) you'd like ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/performance-benchmark.md ================================================ --- name: Performance Benchmark about: Performance benchmark task for feature release title: "[Benchmark] Performance Benchmark for {{ env.RELEASE_VERSION }}" type: "Task" labels: ["release/task", "area/benchmark"] assignees: '' --- ## What's the task? Please describe Execute performance benchmark for v1 and v2 volume ## Describe the sub-tasks - [ ] baseline based on local-path-provisioner - v1 - [ ] 1-replica volume (co-located replica and engine) - [ ] 3-replica volume - v2 - Single CPU core - [ ] 1-replica volume (co-located replica and engine) - [ ] 3-replica volume - Multiple CPU cores - [ ] 1-replica volume (co-located replica and engine) - [ ] 3-replica volume ## Additional context Update the results to https://github.com/longhorn/longhorn/wiki/Performance-Benchmark ================================================ FILE: .github/ISSUE_TEMPLATE/refactor.md ================================================ --- name: Refactor task about: Suggest a refactoring request for an existing implementation title: "[REFACTOR] " type: "Task" labels: kind/refactoring assignees: '' --- ## Is your improvement request related to a feature? Please describe ## Describe the solution you'd like ## Describe alternatives you've considered ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/regular-tasks-for-feature-release.md ================================================ --- name: Regular Tasks for Feature Release about: Regular tasks for a feature release title: "[RELEASE] Regular Tasks for Feature Release {{ env.RELEASE_VERSION }}" type: "Task" labels: ["release/task", "area/install-uninstall-upgrade"] assignees: '' --- ## What's the task? Please describe Regular tasks for feature release {{ env.RELEASE_VERSION }} must be completed well before the release, during the development phase. For OS distro updates for SLES and SLE Micro, need to update for each patch release as well. ## Describe the sub-tasks - [ ] OS Distro Version Update (QA captain) - [ ] Verify by ci.longhorn.io/job/public/job - [ ] Update `Best Practices>Operating System` in the official document and testing pipelines - [ ] K8s Distro Version Update (QA captain) - [ ] Update the testing pipelines to use the latest supported K8s version (RKE2, K3s) - [ ] BCI Image Update for Component Container Base Image - [ ] Golang Version Update - [ ] BCI golang image update - [ ] go.mod update - [ ] Kubernetes Version Update - [ ] Update the official document with all versions we support - [ ] Update the minimum version in official document and chart if needed - [ ] Kubernetes Dependent Library Version Update - [ ] CSI Sidecar Version Update - [ ] Support Bundle Kit Version Update - [ ] NFS-Ganesha Version Update - [ ] SPDK Version Update ## Additional context https://github.com/longhorn/longhorn/wiki/Version-Update-Policy ================================================ FILE: .github/ISSUE_TEMPLATE/release.md ================================================ --- name: Release Task about: Create a release task title: "[RELEASE] Release {{ env.RELEASE_VERSION }}" type: "Task" labels: ["release/task", "area/install-uninstall-upgrade"] assignees: '' --- ## What's the task? Please describe Action items for releasing {{ env.RELEASE_VERSION }} ## Roles - Release captain: {{ env.RELEASE_CAPTAIN }} - QA captain: {{ env.QA_CAPTAIN }} ## Describe the sub-tasks ### Pre-Release #### Release Captain Tasks > [!IMPORTANT] > The Release Captain needs to finish the following items - [ ] This tasks are only needed when doing a feature release such as {{ env.MAJOR_MINOR_VERSION }}. - [ ] Before creating RC1, create a new release branch for the following component repositories by triggering [▶️ Create Longhorn Repository Branches Action](https://github.com/longhorn/release/actions/workflows/create-repo-branches.yml), and then create RC1 from the new branch. Leave the master branch for the next feature release development. - [ ] Add the new branch {{ env.BRANCH_NAME }} to [renovate configuration](https://github.com/longhorn/release/blob/main/renovate-default.json). - [ ] PR: - [ ] After creating the new release branch, update the version file in each repo by [▶️ Update Longhorn Repository Version File in Default Branch Action](https://github.com/longhorn/release/actions/workflows/update-repo-version-file.yml). - longhorn-manager - longhorn-ui - longhorn-tests - longhorn-engine - longhorn-instance-manager - longhorn-share-manager - backing-image-manager - longhorn-spdk-engine (needed after GA) - cli - [ ] Update `jobs.release.strategy.matrix` in [sprint release](https://github.com/longhorn/release/blob/main/.github/workflows/release-sprint.yml). - [ ] PR: - [ ] Trigger the RC release build by [▶️ Release-Preview Action](https://github.com/longhorn/release/actions/workflows/release-preview.yml). #### QA Captain Tasks > [!IMPORTANT] > The QA captain needs to coordinate the following items before the GA release. - [ ] Regression test plan (manual) - [ ] Update Longhorn official document - [ ] Update `Best Practices>Operating System` and `Best Practices>Kubernetes>Kubernetes Version` - [ ] PR: - [ ] Run e2e regression for pre-GA milestones (`install`, `upgrade`) - [ ] Run security testing of container images for pre-GA milestones. - [ ] Investigate and fix the security issues. The issues are tracked by the sub-issue `Fix CVE issues for {{ env.RELEASE_VERSION }}` - @c3y1huang - [ ] Create security issues at upstream for unresolved CVEs in CSI sidecar images - @c3y1huang --- ### Release #### Release Captain Tasks for the GA Build > [!IMPORTANT] > The Release Captain needs to finish the following items - [ ] This tasks are only needed when doing a feature release such as {{ env.MAJOR_MINOR_VERSION }}. - [ ] Ensure the sub-issue `Regular Tasks for Feature Release for {{ env.MAJOR_MINOR_VERSION }}` is completed. - [ ] Ensure the sub-issue `Fix CVE issues for {{ env.RELEASE_VERSION }}` is completed. - [ ] Update image versions in [chart/README.md](https://github.com/longhorn/longhorn/tree/{{ env.RELEASE_VERSION }}/chart/README.md). - PR: - [ ] Trigger the GA release build by [▶️ Release Action](https://github.com/longhorn/release/actions/workflows/release.yml). #### QA Captain Tasks for the GA Build > [!IMPORTANT] > The QA captain needs to coordinate the following items before the GA release. - [ ] Run security testing of container images for GA build - [ ] Verify longhorn chart PR to ensure all artifacts are ready for GA build (`install`, `upgrade`) - [ ] Run core testing (install, upgrade) for the GA build - Upgrade from the previous patch of the same feature release. - Upgrade from the last patch of the previous feature release. #### Release Captain Tasks after Completing the GA Build Validation - [ ] Create a release note ([CHANGELOG](https://github.com/longhorn/longhorn/tree/{{ env.RELEASE_VERSION }}/CHANGELOG)). - [ ] Deprecation note. - PR: - [ ] Update notes including highlighted notes, deprecation, compatible changes, and others impacting the current users. - PR: - [ ] Update [Longhorn official documentation](https://github.com/longhorn/website). - [ ] Update [config.toml](https://github.com/longhorn/website/blob/master/config.toml) and publish the new version of doc and add a next patch version of dev doc. - [ ] Update image versions in `References > Helm Values` and `Snapshot and Backups > CSI Snapshot Support > Enable CSI Snapshot Support on a Cluster`. - PR: - [ ] Update `Important Notes`. - PR: - [ ] Publish the GA release in [longhorn/longhorn](https://github.com/longhorn/longhorn) and [longhorn/cli](https://github.com/longhorn/cli). - [ ] Release longhorn/chart from the release branch to publish to [ArtifactHub](https://artifacthub.io/packages/helm/longhorn/longhorn) by [▶️ Release Charts on Demand Action](https://github.com/longhorn/charts/actions/workflows/release-ondemand.yml). - [ ] Mark the release as `latest` release in longhorn/longhorn [README.md](https://github.com/longhorn/longhorn). - PR: - [ ] Update `jobs.release.strategy.matrix` in [sprint release](https://github.com/longhorn/release/blob/main/.github/workflows/release-sprint.yml). - PR: - [ ] Update Longhorn image tags in longhorn/longhorn/chart/values.yaml in the development branch by triggering [▶️ Update Longhorn Repository Branch Image Tags](https://github.com/longhorn/longhorn/actions/workflows/update-branch-image-tags.yaml). --- ### Post-Release - [ ] Mark the release as `stable` release and update stable versions in https://github.com/longhorn/longhorn/blob/master/support-versions.txt - For the first stable release, we need to consider several factors and reach a consensus by maintainers before claiming it stable. - For any patch release after a stable release, we need to wait 1-2 weeks for user feedback. - PR: **After marking the release as a `stable` release, Release Captain needs to coordinate the following items** - [ ] Update https://github.com/longhorn/longhorn/blob/master/deploy/upgrade_responder_server/chart-values.yaml - @PhanLe1010 - PR: - [ ] Add another request for the rancher charts for the next patch release - @rebeccazzzz - [ ] Update the [support matrix](https://www.suse.com/suse-longhorn/support-matrix/all-supported-versions/) - @asettle @rebeccazzzz - [ ] Update the [lifecycle page](https://www.suse.com/lifecycle/#suse-storage) - @asettle @rebeccazzzz ### Rancher Charts **The Release Captain needs to coordinate the following items.** - [ ] Verify the chart can be installed & upgraded - {{ env.QA_CAPTAIN }} - [ ] rancher/image-mirrors update - @mantissahz @PhanLe1010 - [ ] rancher/charts active branches for Rancher App Marketplace - @mantissahz @PhanLe1010 cc @longhorn/qa @longhorn/dev ================================================ FILE: .github/ISSUE_TEMPLATE/task.md ================================================ --- name: Task about: Create a general task title: "[TASK] " type: "Task" labels: kind/task assignees: '' --- ## What's the task? Please describe ## Describe the sub-tasks ## Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/test.md ================================================ --- name: Test about: Create or update test title: "[TEST] " type: "Test" labels: kind/test assignees: '' --- ## What's the test to develop? Please describe ## Describe the tasks for the test ## Additional context ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ #### Which issue(s) this PR fixes: Issue # #### What this PR does / why we need it: #### Special notes for your reviewer: #### Additional documentation or context ================================================ FILE: .github/mergify.yml ================================================ pull_request_rules: - name: Automatically merge PRs conditions: - check-success=continuous-integration/drone/pr - "#approved-reviews-by>=2" - approved-reviews-by=@longhorn/maintainer actions: merge: method: rebase - name: Automatically merge bot's PRs conditions: - check-success=continuous-integration/drone/pr - or: - author = renovate[bot] - author = mergify[bot] actions: merge: method: rebase - name: Automatically approve bot's PRs conditions: - or: - author = renovate[bot] actions: review: type: APPROVE - name: Ask to resolve conflict conditions: - conflict actions: comment: message: This pull request is now in conflict. Could you fix it @{{author}}? 🙏 ================================================ FILE: .github/workflows/add-issue-to-projects.yml ================================================ name: "[Issue Management] Add Issue To Projects" on: issues: types: [opened, reopened, milestoned] jobs: longhorn: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-issues: write permission-organization-projects: write - name: Check Longhorn Membership (including bot) id: check-membership run: | if [[ "${{ github.event.issue.user.login }}" == "github-actions[bot]" ]]; then echo "is_longhorn_member=true" >> $GITHUB_ENV else echo "is_longhorn_member=false" >> $GITHUB_ENV fi - name: Real Membership Check (non-bot only) if: env.is_longhorn_member == 'false' id: is-longhorn-member continue-on-error: true uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.event.issue.user.login }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Set Final Membership Flag if: env.is_longhorn_member == 'false' && fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null run: echo "is_longhorn_member=true" >> $GITHUB_ENV - name: Debug Membership Status run: | echo "is_longhorn_member: ${{ env.is_longhorn_member }}" echo "teams: ${{ steps.is-longhorn-member.outputs.teams || 'bot or N/A' }}" echo "check outcome: ${{ steps.is-longhorn-member.outcome || 'bot or N/A' }}" - name: Add Issue to Longhorn Sprint Project if: env.is_longhorn_member == 'true' uses: actions/add-to-project@v1.0.2 with: project-url: https://github.com/orgs/longhorn/projects/8 github-token: ${{ steps.app-token.outputs.token }} labeled: kind/test label-operator: NOT community: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-issues: write permission-organization-projects: write - name: Is Longhorn Member id: is-longhorn-member continue-on-error: true uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.event.issue.user.login }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Get Issue if: steps.is-longhorn-member.outcome == 'success' uses: octokit/request-action@v2.x id: issue with: route: GET /repos/${{ github.repository }}/issues/${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Add Issue to Community Sprint Project id: add-project if: | steps.is-longhorn-member.outcome == 'success' && fromJSON(steps.is-longhorn-member.outputs.teams)[0] == null uses: actions/add-to-project@v1.0.2 with: project-url: https://github.com/orgs/longhorn/projects/5 github-token: ${{ steps.app-token.outputs.token }} - name: Update Item To New if: | steps.is-longhorn-member.outcome == 'success' && fromJSON(steps.is-longhorn-member.outputs.teams)[0] == null && steps.add-project.outputs.itemId != '' uses: titoportas/update-project-fields@v0.1.0 with: project-url: https://github.com/orgs/longhorn/projects/5 github-token: ${{ steps.app-token.outputs.token }} item-id: ${{ steps.add-project.outputs.itemId }} field-keys: Status,Sprint field-values: "New,[0]" qa: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-issues: write permission-organization-projects: write - name: Is Longhorn Member id: is-longhorn-member uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.event.issue.user.login }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Add Issue to QA Sprint Project if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null uses: actions/add-to-project@v1.0.2 with: project-url: https://github.com/orgs/longhorn/projects/4 github-token: ${{ steps.app-token.outputs.token }} labeled: kind/test, area/infra label-operator: OR ================================================ FILE: .github/workflows/auto-author-assign.yml ================================================ name: "[PR Management] Auto Author Assign" on: workflow_call: pull_request_target: types: [ opened, reopened ] jobs: assign-author: runs-on: ubuntu-latest steps: - uses: toshimaru/auto-author-assign@v2.1.1 ================================================ FILE: .github/workflows/backport-pr.yml ================================================ name: "[Issue Management] Link Backport PR Issue" on: pull_request: types: [ opened ] branches: - master - "v*" jobs: check-backport: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - name: Check if PR is a backport run: | if [[ "${{ contains(github.event.pull_request.title, 'backport #') }}" == "true" ]]; then echo "BACKPORT=true" >> "$GITHUB_ENV" else echo "BACKPORT=false" >> "$GITHUB_ENV" fi - name: Extract backport branch and issue number if: env.BACKPORT == 'true' run: | # Extract branch from the target branch of the PR BRANCH=$(echo "${{ github.event.pull_request.base.ref }}") BRANCH=${BRANCH%.x} # Remove the '.x' suffix BRANCH=$(echo "${BRANCH}" | sed 's/\./\\./g') # Escape periods echo "BRANCH=$BRANCH" >> $GITHUB_ENV PR_BODY="${{ github.event.pull_request.body }}" ORIGINAL_ISSUE_NUMBER=$(printf '%s\n' "$PR_BODY" \ | grep -oE 'longhorn/longhorn#[0-9]+' \ | head -1 \ | cut -d'#' -f2) if [[ "$ORIGINAL_ISSUE_NUMBER" =~ ^[0-9]+$ ]]; then echo "ORIGINAL_ISSUE_NUMBER=$ORIGINAL_ISSUE_NUMBER" >> $GITHUB_ENV else echo "ORIGINAL_ISSUE_NUMBER=" >> $GITHUB_ENV fi - name: Check if PR is from a fork run: | if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]]; then echo "IS_FORK=true" >> $GITHUB_ENV else echo "IS_FORK=false" >> $GITHUB_ENV fi - id: app-token if: ${{ env.BACKPORT == 'true' && env.IS_FORK == 'true' }} uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-issues: write permission-pull-requests: write - name: Link the PR with the backport issue if: ${{ env.BACKPORT == 'true' && env.IS_FORK == 'true' && env.ORIGINAL_ISSUE_NUMBER != '' }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} run: | for issue_number in "${{ env.ORIGINAL_ISSUE_NUMBER }}"; do echo "Found source issue longhorn/longhorn#${issue_number}" issue_title=$(gh issue view "$issue_number" --repo longhorn/longhorn --json 'title' -q '.title') if [[ $? -eq 0 && -n "$issue_number" ]]; then backport_issue_number=$( curl -s "https://api.github.com/search/issues?q=repo:longhorn/longhorn+is:open+is:issue+in:title+${{ env.BRANCH }}+\"${issue_title}\"" \ | jq .items[0].number ) if [[ -n "$backport_issue_number" ]]; then echo "Found backport issue longhorn/longhorn#${backport_issue_number}" echo "Linking backport issue longhorn/longhorn#${backport_issue_number}" gh issue comment --repo longhorn/longhorn "${backport_issue_number}" --body "Backport PR: ${{ github.event.pull_request.html_url }}" continue fi fi echo "No issue title found" done ================================================ FILE: .github/workflows/check-sprint-last-day.py ================================================ import requests import os import sys from datetime import datetime, timedelta GITHUB_GRAPHQL_URL = "https://api.github.com/graphql" def get_github_project_info(github_token, github_org, github_project): headers = { "Authorization": f"Bearer {github_token}", "Content-Type": "application/json" } query = ''' { organization(login: "%s") { projectsV2(first: 20) { nodes { id title number } } } } ''' % (github_org) payload = { "query": query } response = requests.post(GITHUB_GRAPHQL_URL, headers=headers, json=payload) if response.status_code == 200: # fine project by title nodes = response.json().get("data").get("organization").get("projectsV2").get("nodes") for node in nodes: if node.get("title") == github_project: return node else: response.raise_for_status() def get_current_sprint(github_token, project_id): headers = { "Authorization": f"Bearer {github_token}", "Content-Type": "application/json" } query = ''' query { node(id: "%s") { ... on ProjectV2 { fields(first: 20) { nodes { ... on ProjectV2IterationField { configuration { iterations { startDate id } } } } } } } } ''' % (project_id) payload = { "query": query } response = requests.post(GITHUB_GRAPHQL_URL, headers=headers, json=payload) if response.status_code == 200: # fine project by title result = response.json().get("data").get("node").get("fields").get("nodes") filtered_result = [node for node in result if 'configuration' in node] iterations = filtered_result[0].get("configuration").get("iterations") # Find current iteration current_date = datetime.now().date() current_iteration = None for iteration in iterations: start_date = datetime.strptime(iteration['startDate'], "%Y-%m-%d").date() end_date = start_date + timedelta(days=13) if start_date <= current_date <= end_date: current_iteration = iteration break return current_iteration else: response.raise_for_status() def is_today_is_in_last_day_of_current_sprint(github_token, project_id): current_iteration = get_current_sprint(github_token, project_id) if current_iteration is None: print("Current sprint not found") return False current_date = datetime.now().date() end_date = datetime.strptime(current_iteration['startDate'], "%Y-%m-%d").date() + timedelta(days=13) print("Current date: %s, end date: %s" % (current_date, end_date)) return current_date == end_date if __name__ == "__main__": if len(sys.argv) < 4: print('Usage: python check-sprint-last-day.py github_org github_repo github_project') sys.exit() github_token = os.getenv("GITHUB_TOKEN") github_org = sys.argv[1] github_repo = sys.argv[2] github_project = sys.argv[3] project = get_github_project_info(github_token, github_org, github_project) print(f"GitHub Project Details: {project}") last_day = is_today_is_in_last_day_of_current_sprint(github_token, project.get("id")) if not last_day: print("Today %s is not in last day of current sprint" % datetime.now().date()) sys.exit(1) else: sys.exit(0) ================================================ FILE: .github/workflows/close-issue.yml ================================================ name: "[Issue Management] Close Issue" on: issues: types: [ unlabeled ] jobs: backport: runs-on: ubuntu-latest if: contains(github.event.label.name, 'backport/') steps: - name: Get Backport Version uses: xom9ikk/split@v1 id: split with: string: ${{ github.event.label.name }} separator: / - name: Check if Backport Issue Exists uses: actions-cool/issues-helper@v3 id: if-backport-issue-exists with: actions: 'find-issues' token: ${{ github.token }} title-includes: | [BACKPORT][v${{ steps.split.outputs._1 }}]${{ github.event.issue.title }} - name: Close Backport Issue if: fromJSON(steps.if-backport-issue-exists.outputs.issues)[0] != null uses: actions-cool/issues-helper@v3 with: actions: 'close-issue' token: ${{ github.token }} issue-number: ${{ fromJSON(steps.if-backport-issue-exists.outputs.issues)[0].number }} automation: runs-on: ubuntu-latest if: contains(github.event.label.name, 'require/automation-e2e') steps: - name: Check if Automation Issue Exists uses: actions-cool/issues-helper@v3 id: if-automation-issue-exists with: actions: 'find-issues' token: ${{ github.token }} title-includes: | [TEST]${{ github.event.issue.title }} - name: Close Automation Test Issue if: fromJSON(steps.if-automation-issue-exists.outputs.issues)[0] != null uses: actions-cool/issues-helper@v3 with: actions: 'close-issue' token: ${{ github.token }} issue-number: ${{ fromJSON(steps.if-automation-issue-exists.outputs.issues)[0].number }} ================================================ FILE: .github/workflows/codespell.yml ================================================ name: "[PR Management] Codespell" on: pull_request: branches: - master - "v*.*.*" jobs: codespell: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 1 - name: Check code spell uses: codespell-project/actions-codespell@v2 with: check_filenames: true skip: "*/**.yaml,*/**.yml,./scripts,./vendor,MAINTAINERS,LICENSE,go.mod,go.sum" ================================================ FILE: .github/workflows/conventional-commits.yml ================================================ name: "[PR Management] Conventional PR and Commits" on: pull_request: types: - opened - edited - synchronize - reopened permissions: contents: read pull-requests: read jobs: commit-lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Lint Commits uses: wagoid/commitlint-github-action@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Lint Pull Request uses: amannn/action-semantic-pull-request@v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: types: | feat fix docs style refactor perf test chore vendor build ci revert BREAKING ================================================ FILE: .github/workflows/create-issue.yml ================================================ name: "[Issue Management] Create Backport/Automation/UI Issue" on: issues: types: [ labeled ] env: LONGHORN_SPRINT_PROJECT_URL: https://github.com/orgs/longhorn/projects/8 QA_SPRINT_PROJECT_URL: https://github.com/orgs/longhorn/projects/4 jobs: backport: runs-on: ubuntu-latest if: contains(github.event.label.name, 'backport/') steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-organization-projects: write permission-issues: write - name: Is Longhorn Member uses: tspascoal/get-user-teams-membership@v3 id: is-longhorn-member with: username: ${{ github.actor }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Get Backport Version if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null uses: xom9ikk/split@v1 id: split with: string: ${{ github.event.label.name }} separator: / - name: Check if Backport Issue Exists if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null uses: actions-cool/issues-helper@v3 id: if-backport-issue-exists with: actions: 'find-issues' token: ${{ github.token }} issue-state: 'all' labels: 'kind/backport' issue-creator: 'github-actions[bot]' title-includes: | [BACKPORT][v${{ steps.split.outputs._1 }}]${{ github.event.issue.title }} - name: Get Milestone Object if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-backport-issue-exists.outputs.issues)[0] == null uses: longhorn/bot/milestone-action@master id: milestone with: token: ${{ github.token }} repository: ${{ github.repository }} milestone_name: v${{ steps.split.outputs._1 }} - name: Get Labels if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-backport-issue-exists.outputs.issues)[0] == null id: labels run: | RAW_LABELS="${{ join(github.event.issue.labels.*.name, ' ') }}" RAW_LABELS="${RAW_LABELS} kind/backport" echo "RAW LABELS: $RAW_LABELS" LABELS=$(echo "$RAW_LABELS" | sed -r 's/\s*backport\S+//g' | sed -r 's/\s*require\/auto-e2e-test//g' | xargs | sed 's/ /, /g') echo "LABELS: $LABELS" echo "labels=$LABELS" >> $GITHUB_OUTPUT - name: Get Longhorn Members uses: longhorn/bot/filter-org-members-action@master id: longhorn-members with: token: ${{ steps.app-token.outputs.token }} organization: longhorn usernames: ${{ join(github.event.issue.assignees.*.login, ', ') }} - name: Create Backport Issue if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-backport-issue-exists.outputs.issues)[0] == null uses: dacbd/create-issue-action@main id: create-backport-issue with: token: ${{ github.token }} title: | [BACKPORT][v${{ steps.split.outputs._1 }}]${{ github.event.issue.title }} body: | backport ${{ github.event.issue.html_url }} labels: ${{ steps.labels.outputs.labels }} milestone: ${{ fromJSON(steps.milestone.outputs.data).number }} assignees: ${{ join(fromJSON(steps.longhorn-members.outputs.members), ', ') }} - name: Reopen Backport Issue if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-backport-issue-exists.outputs.issues)[0] != null && fromJSON(steps.if-backport-issue-exists.outputs.issues)[0].state == 'closed' uses: actions-cool/issues-helper@v3 with: actions: 'update-issue' token: ${{ steps.app-token.outputs.token }} issue-number: ${{ fromJSON(steps.if-backport-issue-exists.outputs.issues)[0].number }} state: 'open' - name: Add Issue to Longhorn Sprint Project if: steps.create-backport-issue.outcome == 'success' uses: longhorn/bot/add-to-project-action@master with: project-url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} issue-node-id: ${{ fromJSON(steps.create-backport-issue.outputs.json).node_id }} github-token: ${{ steps.app-token.outputs.token }} automation: runs-on: ubuntu-latest if: contains(github.event.label.name, 'require/auto-e2e-test') steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-organization-projects: write permission-issues: write - name: Is Longhorn Member uses: tspascoal/get-user-teams-membership@v3 id: is-longhorn-member with: username: ${{ github.actor }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Check if Automation Issue Exists if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null uses: actions-cool/issues-helper@v3 id: if-automation-issue-exists with: actions: 'find-issues' token: ${{ github.token }} issue-state: 'all' labels: 'kind/test' issue-creator: 'github-actions[bot]' title-includes: | [TEST]${{ github.event.issue.title }} - name: Create Automation Test Issue if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-automation-issue-exists.outputs.issues)[0] == null uses: dacbd/create-issue-action@main id: create-automation-test-issue with: token: ${{ github.token }} title: | [TEST]${{ github.event.issue.title }} body: | adding/updating auto e2e test cases for ${{ github.event.issue.html_url }} if they can be automated cc @longhorn/qa labels: kind/test - name: Add Issue to QA & Devops Project if: steps.create-automation-test-issue.outcome == 'success' uses: longhorn/bot/add-to-project-action@master with: project-url: ${{ env.QA_SPRINT_PROJECT_URL }} issue-node-id: ${{ fromJSON(steps.create-automation-test-issue.outputs.json).node_id }} github-token: ${{ steps.app-token.outputs.token }} ui: runs-on: ubuntu-latest if: contains(github.event.label.name, 'require/ui') steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-organization-projects: write permission-issues: write - name: Is Longhorn Member uses: tspascoal/get-user-teams-membership@v3 id: is-longhorn-member with: username: ${{ github.actor }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Check if UI Issue Exists if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null uses: actions-cool/issues-helper@v3 id: if-ui-issue-exists with: actions: 'find-issues' token: ${{ steps.app-token.outputs.token }} issue-state: 'all' labels: 'area/ui' issue-creator: 'github-actions[bot]' title-includes: | [UI]${{ github.event.issue.title }} - name: Get Labels if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-ui-issue-exists.outputs.issues)[0] == null id: labels run: | RAW_LABELS="${{ join(github.event.issue.labels.*.name, ' ') }}" echo "RAW LABELS: $RAW_LABELS" LABELS=$(echo "$RAW_LABELS" | sed -r 's/\s*backport\S+//g' | sed -r 's/\s*require\/ui//g' | xargs | sed 's/ /, /g') echo "LABELS: $LABELS" echo "labels=$LABELS" >> $GITHUB_OUTPUT - name: Create UI Issue if: fromJSON(steps.is-longhorn-member.outputs.teams)[0] != null && fromJSON(steps.if-ui-issue-exists.outputs.issues)[0] == null uses: dacbd/create-issue-action@main id: create-ui-issue with: token: ${{ github.token }} title: | [UI]${{ github.event.issue.title }} body: | make corresponding UI changes for ${{ github.event.issue.html_url }} cc @longhorn/dev-ui labels: ${{ steps.labels.outputs.labels }}, area/ui milestone: ${{ github.event.issue.milestone.number }} - name: Add Issue to Longhorn Sprint Project if: steps.create-ui-issue.outcome == 'success' uses: longhorn/bot/add-to-project-action@master with: project-url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} issue-node-id: ${{ fromJSON(steps.create-ui-issue.outputs.json).node_id }} github-token: ${{ steps.app-token.outputs.token }} ================================================ FILE: .github/workflows/create-release-task-issues.yml ================================================ name: "[Release] Create Release Task Issues" on: workflow_dispatch: inputs: release_version: description: "Release Version, e.g. v1.8.1" required: true release_captain: description: "GitHub Username of the Release Captain, formatted as @" required: true qa_captain: description: "GitHub Username of the QA Captain, formatted as @" required: true jobs: create-release-task: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-issues: write - name: Checkout code uses: actions/checkout@v3 - name: Parse version information id: parse_version run: | set -e RELEASE_VERSION="${{ github.event.inputs.release_version }}" # Validate version format if ! echo "$RELEASE_VERSION" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then echo "Error: Invalid release version format. Expected format: v.. (e.g., v1.8.1)." >&2 exit 1 fi MAJOR_MINOR_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/(v[0-9]+\.[0-9]+)\.[0-9]+/\1.0/') BRANCH_NAME=$(echo "$RELEASE_VERSION" | sed -E 's/(v[0-9]+\.[0-9]+)\.[0-9]+/\1.x/') FEATURE_RELEASE="false" if echo "$RELEASE_VERSION" | grep -Eq '^v[0-9]+\.[0-9]+\.0$'; then FEATURE_RELEASE="true" fi echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV echo "MAJOR_MINOR_VERSION=$MAJOR_MINOR_VERSION" >> $GITHUB_ENV echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV echo "FEATURE_RELEASE=$FEATURE_RELEASE" >> $GITHUB_ENV - name: Create an issue - release task uses: rancher/gh-issue-mgr/create-an-issue@main with: filename: .github/ISSUE_TEMPLATE/release.md env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} RELEASE_VERSION: ${{ env.RELEASE_VERSION }} MAJOR_MINOR_VERSION: ${{ env.MAJOR_MINOR_VERSION }} BRANCH_NAME: ${{ env.BRANCH_NAME }} FEATURE_RELEASE: ${{ env.FEATURE_RELEASE }} RELEASE_CAPTAIN: ${{ github.event.inputs.release_captain }} QA_CAPTAIN: ${{ github.event.inputs.qa_captain }} - name: Create an issue - fix security issue for release uses: rancher/gh-issue-mgr/create-an-issue@main with: filename: .github/ISSUE_TEMPLATE/fix-cve-issues-for-release.md env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} RELEASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Create an issue - regular tasks for feature release if: ${{ env.FEATURE_RELEASE == 'true' }} uses: rancher/gh-issue-mgr/create-an-issue@main with: filename: .github/ISSUE_TEMPLATE/regular-tasks-for-feature-release.md env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} RELEASE_VERSION: ${{ env.RELEASE_VERSION }} FEATURE_RELEASE: ${{ env.FEATURE_RELEASE }} BRANCH_NAME: ${{ env.BRANCH_NAME }} - name: Create an issue - performance benchmark task for feature release if: ${{ env.FEATURE_RELEASE == 'true' }} uses: rancher/gh-issue-mgr/create-an-issue@main with: filename: .github/ISSUE_TEMPLATE/performance-benchmark.md env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} RELEASE_VERSION: ${{ env.RELEASE_VERSION }} FEATURE_RELEASE: ${{ env.FEATURE_RELEASE }} ================================================ FILE: .github/workflows/periodic-issue-sprint-update.yml ================================================ name: "[Issue Management] Periodic Issue Sprint Update" on: schedule: # Trigger every Sunday at 20:00 - cron: '0 20 * * 0' workflow_dispatch: permissions: contents: read repository-projects: write issues: write jobs: move-to-next-iteration: name: Move to next iteration runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-issues: write permission-organization-projects: write - name: Checkout repository uses: actions/checkout@v4 - name: Check sprint build is required id: check_sprint_build_required continue-on-error: true run: | python ./.github/workflows/check-sprint-last-day.py "longhorn" "longhorn" "Longhorn Sprint" - name: Longhorn Sprint Issues - Clear Sprint uses: rancher/gh-issue-mgr/move-to-next-iteration@main if: steps.check_sprint_build_required.outcome == 'success' with: owner: longhorn number: 8 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: current new-iteration: none excluded-statuses: "Review,Ready For Testing,Testing,Closed" - name: Longhorn Sprint Issues - Move to Next Sprint uses: rancher/gh-issue-mgr/move-to-next-iteration@main if: steps.check_sprint_build_required.outcome == 'success' with: owner: longhorn number: 8 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: current new-iteration: next statuses: "Review" - name: Longhorn Sprint Issues - Update None to Current Sprint if: steps.check_sprint_build_required.outcome == 'success' uses: rancher/gh-issue-mgr/move-to-next-iteration@main with: owner: longhorn number: 8 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: none new-iteration: current statuses: "Ready For Testing" - name: QA Sprint - Clear Sprint uses: rancher/gh-issue-mgr/move-to-next-iteration@main if: steps.check_sprint_build_required.outcome == 'success' with: owner: longhorn number: 4 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: current new-iteration: none excluded-statuses: "Review,Testing,Closed" - name: QA Sprint - Move to Next Sprint uses: rancher/gh-issue-mgr/move-to-next-iteration@main if: steps.check_sprint_build_required.outcome == 'success' with: owner: longhorn number: 4 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: current new-iteration: next statuses: "Review" - name: Longhorn Community Sprint issues - Move to Next Sprint uses: rancher/gh-issue-mgr/move-to-next-iteration@main if: steps.check_sprint_build_required.outcome == 'success' with: owner: longhorn number: 5 token: ${{ steps.app-token.outputs.token }} iteration-field: Sprint iteration: current new-iteration: next statuses: 'New' ================================================ FILE: .github/workflows/pr-review-reminder.py ================================================ import subprocess import json import os import requests REPOS = ["longhorn/longhorn-manager", "longhorn/longhorn-engine", "longhorn/longhorn-instance-manager", "longhorn/longhorn-share-manager", "longhorn/backing-image-manager", "longhorn/longhorn-ui", "longhorn/longhorn-spdk-engine", "longhorn/go-iscsi-helper", "longhorn/go-spdk-helper", "longhorn/backupstore", "longhorn/go-common-libs", "longhorn/types"] def flatten_issues(repo, blocks, issues, user_mapping): # Append the title and divider only if there are issues to display print(issues) if issues: blocks.append( { "type": "section", "text": { "type": "mrkdwn", "text": f"*{repo}* - {len(issues)} prs" } } ) blocks.append({"type": "divider"}) # Combine issues into chunks of 5 issue_texts = [] for i, issue in enumerate(issues): number = issue["number"] title = issue["title"] issue_url = f"https://github.com/{repo}/pull/{number}" reviewers = [] for reviewer in issue["reviewers"]: slack_id = user_mapping.get(reviewer) if not slack_id: reviewers.append(reviewer) else: reviewers.append(f"<@{slack_id}>") issue_texts.append(f"- *<{issue_url}|{number}>* - {title} - {', '.join(reviewers)}") # Add a block for every 5 issues for avoiding bad request error if (i + 1) % 5 == 0 or (i + 1) == len(issues): blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": "\n".join(issue_texts) # Combine all issue texts } }) issue_texts = [] # Reset for the next chunk return blocks def send_slack_notification(issues): github_token = os.getenv("GITHUB_TOKEN") webhook_url = os.getenv("SLACK_WEBHOOK_URL") value = os.getenv("USER_MAPPING") user_mapping = {} if value is not None: user_mapping = json.loads(value) print("Sending Slack notification...") # Initialize blocks as an empty list blocks = [] blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": "Hello, this is a Pull Request Review reminder. \n\n" + "Please help review the following Pull Requests. Thanks for your efforts!" } }) for repo in REPOS: print(f"Processing prs in {repo}...") blocks = flatten_issues(repo, blocks, issues[repo], user_mapping) payload = { "blocks": blocks } headers = { 'Content-Type': 'application/json' } print(f"Payload: {json.dumps(payload, indent=2)}") response = requests.post(webhook_url, json=payload, headers=headers) response.raise_for_status() def pr_review_reminder(): """ Fetches and prints details of open Pull Requests for a given repository. """ issues = {repo: [] for repo in REPOS} for repo in REPOS: print(f"Checking PRs in {repo}...") # Construct the gh command command = [ "gh", "pr", "list", "--repo", repo, "--state", "open", "--json", "number,title,author,reviewRequests,labels" ] try: # Run the command result = subprocess.run(command, capture_output=True, text=True, check=False) # Check if the command was successful if result.returncode != 0: print(f" Error running gh command for {repo}:") if result.stdout: print(f" Stdout: {result.stdout.strip()}") if result.stderr: print(f" Stderr: {result.stderr.strip()}") continue # Check if there's any output to parse if not result.stdout.strip(): print(f" No open PRs found or no output from gh for {repo}.") continue # Parse the JSON output prs_data = json.loads(result.stdout) if not prs_data: print(f" No open PRs found in {repo}.") continue # Iterate through each PR for pr in prs_data: pr_number = pr.get("number", "N/A") pr_title = pr.get("title", "N/A") pr_reviewers = pr.get("reviewRequests", []) labels = pr.get("labels", []) # Skip PR if its author is a bot if pr.get("author", {}).get("is_bot", False): print(f" Skipping PR #{pr_number} by bot author: {pr.get('author', {}).get('login', 'Unknown')}") continue # Skip PR if it has "pending" label if any(label.get("name") == "pending" for label in labels): print(f" Skipping PR #{pr_number} due to 'pending' label.") continue # Get author login author_info = pr.get("author") pr_author = "Unknown Author" if isinstance(author_info, dict): pr_author = author_info.get("login", "Unknown Author") # Add PR details to issues list and classify using repos issues[repo].append({ "number": pr_number, "title": pr_title, "author": pr_author, "reviewers": [reviewer.get("login", "Unknown") for reviewer in pr_reviewers], }) except Exception as e: print(f" An unexpected error occurred while processing {repo}: {e}") finally: print() send_slack_notification(issues) if __name__ == "__main__": pr_review_reminder() ================================================ FILE: .github/workflows/pr-review-reminder.yml ================================================ name: "[Issue Management] Pull Request Review Reminder" on: workflow_dispatch: schedule: # Trigger every Monday at 10:00 GMT+8 - cron: '0 2 * * 1' jobs: pull-request-review-reminder: runs-on: ubuntu-latest name: "Pull Request Review Reminder" steps: - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install requests - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: read permission-issues: read permission-pull-requests: read permission-organization-projects: read - name: Authenticate GitHub CLI run: echo "${{ steps.app-token.outputs.token }}" | gh auth login --with-token - uses: actions/checkout@v4 - name: Pull Request Review Reminder env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_LONGHORN_QA_WEBHOOK_URL }} USER_MAPPING: ${{ secrets.USER_MAPPING_FOR_GITHUB_SLACK }} run: python .github/workflows/pr-review-reminder.py ================================================ FILE: .github/workflows/scan-and-notify-testing-items.py ================================================ import requests import os import sys import time import json from datetime import datetime, timedelta GITHUB_GRAPHQL_URL = "https://api.github.com/graphql" def get_github_project_info(github_token, github_org, github_project): headers = { "Authorization": f"Bearer {github_token}", "Content-Type": "application/json" } query = ''' { organization(login: "%s") { projectsV2(first: 20) { nodes { id title number } } } } ''' % (github_org) payload = { "query": query } response = requests.post(GITHUB_GRAPHQL_URL, headers=headers, json=payload) if response.status_code == 200: # Find project by title print("Response: %s" % response.json()) nodes = response.json().get("data").get("organization").get("projectsV2").get("nodes") for node in nodes: if node.get("title") == github_project: return node else: response.raise_for_status() def get_current_sprint(github_token, project_id): headers = { "Authorization": f"Bearer {github_token}", "Content-Type": "application/json" } query = ''' query { node(id: "%s") { ... on ProjectV2 { fields(first: 20) { nodes { ... on ProjectV2IterationField { configuration { iterations { startDate id } } } } } } } } ''' % (project_id) payload = { "query": query } response = requests.post(GITHUB_GRAPHQL_URL, headers=headers, json=payload) if response.status_code == 200: # Find project by title result = response.json().get("data").get("node").get("fields").get("nodes") filtered_result = [node for node in result if 'configuration' in node] iterations = filtered_result[0].get("configuration").get("iterations") # Find current iteration current_date = datetime.now().date() current_iteration = None for iteration in iterations: start_date = datetime.strptime(iteration['startDate'], "%Y-%m-%d").date() end_date = start_date + timedelta(days=13) if start_date <= current_date <= end_date: current_iteration = iteration break return current_iteration else: response.raise_for_status() def is_today_is_in_last_day_of_current_sprint(github_token, project_id): current_iteration = get_current_sprint(github_token, project_id) if current_iteration is None: print("Current sprint not found") return False current_date = datetime.now().date() end_date = datetime.strptime(current_iteration['startDate'], "%Y-%m-%d").date() + timedelta(days=13) return current_date == end_date def list_issues_in_project(github_token, project_id, desired_status=None): headers = { "Authorization": f"bearer {github_token}", "Content-Type": "application/json" } query = """ query($project: ID!, $cursor: String) { node(id: $project) { ... on ProjectV2 { items(first: 100, after: $cursor) { nodes { content { ... on Issue { number title assignees(first: 10) { nodes { login } } } } status: fieldValueByName(name: "Status") { ... on ProjectV2ItemFieldSingleSelectValue { name } } sprint: fieldValueByName(name: "Sprint") { ... on ProjectV2ItemFieldIterationValue { title startDate } } } pageInfo { endCursor hasNextPage } } } } } """ cursor = None current_issues = [] non_current_issues = [] current_sprint = get_current_sprint(github_token, project_id) print(f"Current sprint: {current_sprint}") while True: variables = {"project": project_id, "cursor": cursor} response = requests.post(GITHUB_GRAPHQL_URL, headers=headers, json={"query": query, "variables": variables}) if response.status_code == 200: data = response.json() items = data['data']['node']['items']['nodes'] for item in items: status = item['status']['name'] if desired_status and status not in desired_status: continue sprint = item['sprint'] if not sprint or not sprint.get('startDate') or not current_sprint or sprint['startDate'] != current_sprint['startDate']: non_current_issues.append(item) else: current_issues.append(item) page_info = data['data']['node']['items']['pageInfo'] if page_info['hasNextPage']: cursor = page_info['endCursor'] else: break else: raise Exception(f"Query failed to run by returning code of {response.status_code}. {response.text}") return current_issues, non_current_issues def flatten_issues(title, blocks, issues, user_mapping): # Append the title and divider only if there are issues to display if issues: blocks.append( { "type": "section", "text": { "type": "mrkdwn", "text": f"*{title}* - {len(issues)} issues" } } ) blocks.append({"type": "divider"}) # Combine issues into chunks of 5 issue_texts = [] for i, issue in enumerate(issues): number = issue["content"]["number"] title = issue["content"]["title"] issue_url = f"https://github.com/longhorn/longhorn/issues/{number}" assignees = [] for assignee in issue["content"]["assignees"]["nodes"]: slack_id = user_mapping.get(assignee["login"]) if not slack_id: assignees.append(assignee["login"]) else: assignees.append(f"<@{slack_id}>") issue_texts.append(f"- *<{issue_url}|{number}>* - {title} - {', '.join(assignees)}") # Add a block for every 5 issues for avoiding bad request error if (i + 1) % 5 == 0 or (i + 1) == len(issues): blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": "\n".join(issue_texts) # Combine all issue texts } }) issue_texts = [] # Reset for the next chunk return blocks def send_slack_notification(webhook_url, user_mapping, current_issues, non_current_issues): if len(current_issues) == 0 and len(non_current_issues) == 0: print("Nothing to notify") return # Initialize blocks as an empty list blocks = [] blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": "Hello , this is a reminder. \n\n" + "There are 'Ready for Testing' or 'Testing' issues. Please finish verifying them using the corresponding sprint release soon. \n\n" + " - If passed, move them to 'Closed' and DO NOT change the sprint. \n" + " - If not passed, move them to 'Implementation' and update the sprint to the current one. \n\n" + "Thanks for your efforts!" } }) blocks = flatten_issues("Ready for Testing Issues from the Previous Sprint", blocks, current_issues, user_mapping) blocks = flatten_issues("Ready for Testing Issues from older Sprints", blocks, non_current_issues, user_mapping) payload = { "blocks": blocks } headers = { 'Content-Type': 'application/json' } response = requests.post(webhook_url, json=payload, headers=headers) response.raise_for_status() def scan_and_notify(github_org, github_repo, github_project): github_token = os.getenv("GITHUB_TOKEN") webhook_url = os.getenv("SLACK_WEBHOOK_URL") value = os.getenv("USER_MAPPING") user_mapping = {} if value is not None: user_mapping = json.loads(value) project = get_github_project_info(github_token, github_org, github_project) print(f"GitHub Project Details: {project}") # last_day = is_today_is_in_last_day_of_current_sprint(github_token, project.get("id")) # if not last_day: # print("Today %s is not in last day of current sprint" % datetime.now().date()) # return project_id = project.get("id") current_issues, non_current_issues = list_issues_in_project(github_token, project_id, ["Ready For Testing", "Testing"]) print("Number of \"Ready For Testing\" and \"Testing\" issues for current sprint:", len(current_issues)) print("Number of \"Ready For Testing\" and \"Testing\" issues for non-current sprint:", len(non_current_issues)) send_slack_notification(webhook_url, user_mapping, current_issues, non_current_issues) if __name__ == "__main__": if len(sys.argv) < 4: print('Usage: python scan-and-notify-testing-items.py ') scan_and_notify(sys.argv[1], sys.argv[2], sys.argv[3]) ================================================ FILE: .github/workflows/scan-and-notify-testing-items.yml ================================================ name: "[Notification] Scan and Notify Testing Items" on: schedule: # Trigger every Sunday at 12:00 - cron: '0 12 * * 0' workflow_dispatch: inputs: github_org: description: "GitHub organization" default: "longhorn" required: true github_repo: description: "GitHub repository" default: "longhorn" required: true github_project: description: "GitHub project" default: "Longhorn Sprint" required: true env: GITHUB_ORG: longhorn GITHUB_REPO: longhorn GITHUB_PROJECT: Longhorn Sprint jobs: scan_and_notify_testing_items: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: read permission-issues: read - name: Checkout repository uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install requests - name: Scan and Notify Testing Items env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_LONGHORN_QA_WEBHOOK_URL }} USER_MAPPING: ${{ secrets.USER_MAPPING_FOR_GITHUB_SLACK }} run: | github_org="${GITHUB_ORG}" github_repo="${GITHUB_REPO}" github_project="${GITHUB_PROJECT}" if [ -n "${{ github.event.inputs.github_org }}" ]; then github_org="${{ github.event.inputs.github_org }}" fi if [ -n "${{ github.event.inputs.github_repo }}" ]; then github_repo="${{ github.event.inputs.github_repo }}" fi if [ -n "${{ github.event.inputs.github_project }}" ]; then github_project="${{ github.event.inputs.github_project }}" fi python ./.github/workflows/scan-and-notify-testing-items.py "${github_org}" "${github_repo}" "${github_project}" ================================================ FILE: .github/workflows/scorecards.yml ================================================ name: "[Security] Scorecard Supply-Chain Security" on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '30 3 * * 4' push: branches: [ "master" ] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecard analysis runs-on: ubuntu-latest permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. # contents: read # actions: read steps: - name: "Checkout code" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 with: results_file: results.sarif results_format: sarif # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. # repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers # - Allows the repository to include the Scorecard badge. # - See https://github.com/ossf/scorecard-action#publishing-results. # For private repositories: # - `publish_results` will always be set to `false`, regardless # of the value entered here. publish_results: true # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@v4 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@v3 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/stale.yaml ================================================ name: "[Issue Management] Close Stale Issues and PRs" on: workflow_call: workflow_dispatch: schedule: - cron: '30 1 * * *' jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v9 with: stale-issue-message: 'This issue is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 10 days.' close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' close-pr-message: 'This PR was closed because it has been stalled for 5 days with no activity.' close-issue-label: 'wontfix' days-before-stale: 30 days-before-pr-stale: 30 days-before-close: 5 days-before-pr-close: 5 stale-issue-label: 'stale' stale-pr-label: 'stale' exempt-all-assignees: true exempt-issue-labels: 'kind/improvement,kind/feature,kind/test,investigation-needed' exempt-draft-pr: true exempt-all-milestones: true ================================================ FILE: .github/workflows/update-branch-image-tags.yaml ================================================ name: "[Release] Update Longhorn Repository Branch Image Tags" on: workflow_dispatch: inputs: branch: description: "Branch, ex: v1.7.x" required: true defaults: run: shell: bash jobs: update-repo-branch-image-tags: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} - name: Prepare Packages run: sudo apt update -y ; sudo apt install -y snapd; sudo snap install yq - name: Update Repo Branch Image Tags in deploy/longhorn-images.txt run: | set -o errexit set -o xtrace tag=${{ inputs.branch }}-head repos_dir=.repos images=( longhornio/backing-image-manager longhornio/longhorn-engine longhornio/longhorn-instance-manager longhornio/longhorn-manager longhornio/longhorn-share-manager longhornio/longhorn-ui longhornio/longhorn-cli ) function replace_images_tags() { local input_file="$1" local tag="$2" local output_file="${input_file}.new" if [ -z "$input_file" ] || [ -z "$tag" ]; then echo "Usage: replace_longhorn_images " return 1 fi while IFS= read -r line; do modified=false for img in "${images[@]}"; do if [[ "$line" == *"$img"* ]]; then if [[ "$line" =~ $img(:[^ ]*)? ]]; then line=$(echo "$line" | sed -E "s|$img(:[^ ]*)?|$img:$tag|") modified=true break fi fi done echo "$line" >> "$output_file" done < "$input_file" if [ $? -eq 0 ]; then mv "$output_file" "$input_file" echo "Successfully replaced Longhorn image tags in '$input_file'." else rm -f "$output_file" echo "Error: Failed to replace Longhorn image tags." return 1 fi } replace_images_tags "deploy/longhorn-images.txt" "$tag" - name: Update Repo Branch Image Tags in chart/values.yaml run: | yq eval '.image.longhorn.engine.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml yq eval '.image.longhorn.manager.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml yq eval '.image.longhorn.ui.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml yq eval '.image.longhorn.instanceManager.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml yq eval '.image.longhorn.shareManager.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml yq eval '.image.longhorn.backingImageManager.tag = "${{ inputs.branch }}-head"' -i chart/values.yaml - name: Regenerate Manifests run: bash ./scripts/generate-longhorn-yaml.sh - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@v7 with: token: ${{ steps.app-token.outputs.token }} branch: "update-image-tags-${{ inputs.branch }}" delete-branch: true sign-commits: true signoff: true author: Longhorn GitHub Bot <67932897+longhorn-io-github-bot@users.noreply.github.com> committer: Longhorn GitHub Bot <67932897+longhorn-io-github-bot@users.noreply.github.com> commit-message: "chore: update image tags for branch ${{ inputs.branch }}" title: "chore: update image tags for branch ${{ inputs.branch }}" body: | This PR updates the image tags for branch ${{ inputs.branch }} to ${{ inputs.branch }}-head. - name: Enable Pull Request Automerge if: steps.cpr.outputs.pull-request-operation == 'created' uses: peter-evans/enable-pull-request-automerge@v3 with: token: ${{ steps.app-token.outputs.token }} pull-request-number: ${{ steps.cpr.outputs.pull-request-number }} merge-method: rebase ================================================ FILE: .github/workflows/update-community-issue.yml ================================================ name: "[Issue Management] Update Community Issue Status, Sprint and Milestone" on: issues: types: [labeled, milestoned, reopened, closed] issue_comment: types: [created, edited] concurrency: group: ${{ github.workflow }}-${{ github.event.issue.number }} cancel-in-progress: true jobs: community: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-issues: write permission-organization-projects: write - name: Checkout Repository uses: actions/checkout@v4 - name: Is Longhorn Member id: is-longhorn-member continue-on-error: true uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.event.issue.user.login }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - run: | echo "is-longhorn-member teams: ${{ steps.is-longhorn-member.outputs.teams }}" echo "is-longhorn-member outcome: ${{ steps.is-longhorn-member.outcome }}" - name: Get Issue uses: octokit/request-action@v2.x id: issue if: steps.is-longhorn-member.outcome == 'success' with: route: GET /repos/${{ github.repository }}/issues/${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - uses: EndBug/project-fields@v2 id: fields continue-on-error: true if: steps.is-longhorn-member.outcome == 'success' with: operation: get fields: Status github_token: ${{ steps.app-token.outputs.token }} project_url: https://github.com/orgs/longhorn/projects/5 resource_url: https://github.com/longhorn/longhorn/issues/${{ github.event.issue.number }} - run: echo ${{ steps.fields.outputs.values }} - name: Add Issue to Community Sprint Project id: add-project uses: actions/add-to-project@v1.0.2 if: ${{ steps.is-longhorn-member.outcome == 'success' && steps.fields.outcome == 'success' }} with: project-url: https://github.com/orgs/longhorn/projects/5 github-token: ${{ steps.app-token.outputs.token }} - name: Update Project Item if: ${{ steps.is-longhorn-member.outcome == 'success' && steps.fields.outcome == 'success' && steps.add-project.outputs.itemId != '' }} run: | teams=${{ steps.is-longhorn-member.outputs.teams }} teams=${teams:1:-1} IFS=',' read -r -a team_array <<< "$teams" team_count=${#team_array[@]} echo "team_count: $team_count" # If the issue is milestoned by longhorn member, update to Resolved if [[ $team_count -gt 0 && "${{ github.event_name }}" == "issues" && "${{ github.event.action }}" == "milestoned" ]]; then echo "Updating to Resolved since it's milestoned by longhorn member" echo "field-values=Resolved,[0]" >> $GITHUB_ENV # If the issue is closed, update to Closed elif [[ "${{ github.event_name }}" == "issues" && "${{ github.event.action }}" == "closed" && "${{ steps.fields.outputs.values }}" != "Resolved" ]]; then echo "Updating to Closed since it's closed" echo "field-values=Closed" >> $GITHUB_ENV # If the issue is reopened with status Closed, update to In Progress elif [[ "${{ github.event_name }}" == "issues" && "${{ github.event.action }}" == "reopened" && "${{ steps.fields.outputs.values }}" == "Closed" ]]; then echo "Updating to In Progress" echo "field-values=In Progress,[0]" >> $GITHUB_ENV # If a comment is added or edited and the status is not Resolved or Closed by non-longhorn member, update to In Progress elif [[ $team_count -eq 0 && "${{ github.event_name }}" == "issue_comment" && ( "${{ github.event.action }}" == "created" || "${{ github.event.action }}" == "edited" ) && ( "${{ steps.fields.outputs.values }}" != "Resolved" && "${{ steps.fields.outputs.values }}" != "Closed" ) ]]; then echo "Updating to In Progress" echo "field-values=In Progress,[0]" >> $GITHUB_ENV # If the issue is labeled as invalid, wontfix or duplicated, update to Closed elif [[ "${{ github.event_name }}" == "issues" && "${{ github.event.action }}" == "labeled" && ( "${{ github.event.label.name }}" == "invalid" || "${{ github.event.label.name }}" == "wontfix" || "${{ github.event.label.name }}" == "duplicated" ) ]]; then echo "Updating to Closed" echo "field-values=Closed" >> $GITHUB_ENV else echo "No matching conditions, skipping update." exit 0 fi env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Apply Project Update if: ${{ steps.is-longhorn-member.outcome == 'success' && steps.fields.outcome == 'success' && env.field-values != '' }} uses: titoportas/update-project-fields@v0.1.0 with: project-url: https://github.com/orgs/longhorn/projects/5 github-token: ${{ steps.app-token.outputs.token }} item-id: ${{ steps.add-project.outputs.itemId }} field-keys: Status,Sprint field-values: ${{ env.field-values }} - name: Add Resolved Issue to Longhorn Sprint Project if: ${{ steps.is-longhorn-member.outcome == 'success' && steps.fields.outcome == 'success' && env.field-values != '' && startsWith(env.field-values, 'Resolved') }} uses: actions/add-to-project@v1.0.2 with: project-url: https://github.com/orgs/longhorn/projects/8 github-token: ${{ steps.app-token.outputs.token }} - name: Set Milestone to Backlog if: ${{ steps.is-longhorn-member.outcome == 'success' && steps.fields.outcome == 'success' && env.field-values != '' && startsWith(env.field-values, 'Resolved') && github.event.issue.milestone == null }} run: | echo "Setting milestone $MILESTONE to Backlog" gh issue edit "${{ github.event.issue.number }}" --milestone "Backlog" env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} ================================================ FILE: .github/workflows/update-longhorn-issue.yml ================================================ name: "[Issue Management] Update Issue Status in Longhorn Sprint Project" on: issues: types: [labeled, milestoned, assigned, unassigned] env: LONGHORN_SPRINT_PROJECT_URL: https://github.com/orgs/longhorn/projects/8 concurrency: group: ${{ github.workflow }}-${{ github.event.issue.number }} cancel-in-progress: true jobs: longhorn: runs-on: ubuntu-latest steps: - id: app-token uses: actions/create-github-app-token@v2 with: app-id: ${{ secrets.LONGHORN_GITHUB_BOT_APP_ID }} private-key: ${{ secrets.LONGHORN_GITHUB_BOT_PRIVATE_KEY }} owner: ${{ github.repository_owner }} permission-contents: write permission-pull-requests: write permission-members: read permission-issues: write permission-organization-projects: write - name: Checkout Repository uses: actions/checkout@v4 - name: Check Longhorn Membership (including bot) id: check-membership run: | if [[ "${{ github.event.issue.user.login }}" == "github-actions[bot]" ]]; then echo "is_longhorn_member=true" >> $GITHUB_ENV else echo "is_longhorn_member=false" >> $GITHUB_ENV fi - name: Real Membership Check (non-bot only) if: env.is_longhorn_member == 'false' id: is-longhorn-member continue-on-error: true uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.event.issue.user.login }} organization: longhorn GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - name: Set Final Membership Flag if: env.is_longhorn_member == 'false' run: | if [[ "${{ steps.is-longhorn-member.outcome }}" == "success" ]]; then echo "is_longhorn_member=true" >> $GITHUB_ENV fi - name: Debug Membership Status run: | echo "is_longhorn_member: ${{ env.is_longhorn_member }}" echo "teams: ${{ steps.is-longhorn-member.outputs.teams || 'bot or N/A' }}" echo "check outcome: ${{ steps.is-longhorn-member.outcome || 'bot or N/A' }}" - name: Get Issue uses: octokit/request-action@v2.x id: issue if: env.is_longhorn_member == 'true' with: route: GET /repos/${{ github.repository }}/issues/${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - uses: rancher/gh-issue-mgr/project-fields@main id: get-status-field continue-on-error: true if: env.is_longhorn_member == 'true' with: operation: get fields: Status github_token: ${{ steps.app-token.outputs.token }} project_url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} resource_url: https://github.com/longhorn/longhorn/issues/${{ github.event.issue.number }} - name: Add Issue to Longhorn Sprint Project id: add-project uses: actions/add-to-project@v1.0.2 if: ${{ env.is_longhorn_member == 'true' && steps.get-status-field.outcome == 'success' }} with: project-url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} github-token: ${{ steps.app-token.outputs.token }} - name: Update Item To New if: | env.is_longhorn_member == 'true' && steps.add-project.outputs.itemId != '' uses: titoportas/update-project-fields@v0.1.0 with: project-url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} github-token: ${{ steps.app-token.outputs.token }} item-id: ${{ steps.add-project.outputs.itemId }} field-keys: Status field-values: "New" - name: Set Milestone Backlog to Issue if: ${{ env.is_longhorn_member == 'true' && fromJSON(steps.is-longhorn-member.outputs.teams || '[]')[0] != null && github.event.issue.milestone == null && steps.get-status-field.outcome == 'success' }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} run: | ISSUE_NUMBER="${{ github.event.issue.number }}" # Reduce the chance that milestone is modified by users and actions at the same time. MILESTONE_TITLE=$(gh issue view "$ISSUE_NUMBER" --json milestone --jq '.milestone.title') if [ -z "$MILESTONE_TITLE" ]; then gh issue edit "$ISSUE_NUMBER" --milestone "Backlog" fi - name: Update Project Item if: ${{ env.is_longhorn_member == 'true' && steps.get-status-field.outcome == 'success' && steps.add-project.outputs.itemId != '' }} run: | ASSIGNEES='${{ toJSON(github.event.issue.assignees) }}' # Skip if status is already Closed if [[ "${{ steps.get-status-field.outputs.values }}" == "Closed" ]]; then echo "Status is Closed, skipping update" exit 0 fi if [[ ( "${{ steps.get-status-field.outputs.values }}" == "New Issues" || "${{ steps.get-status-field.outputs.values }}" == "" ) && $ASSIGNEES != "[]" ]]; then echo "Updating to New Issues since it's assigned and status is New Issues or empty" echo "field-values=New Issues" >> $GITHUB_ENV elif [[ $ASSIGNEES == "[]" ]]; then echo "Updating to New Issues since it's unassigned" echo "field-values=New Issues" >> $GITHUB_ENV fi - name: Apply Project Update if: ${{ env.is_longhorn_member == 'true' && steps.get-status-field.outcome == 'success' && env.field-values != '' }} uses: rancher/gh-issue-mgr/update-project-fields@main with: project-url: ${{ env.LONGHORN_SPRINT_PROJECT_URL }} github-token: ${{ steps.app-token.outputs.token }} item-id: ${{ steps.add-project.outputs.itemId }} field-keys: Status field-values: ${{ env.field-values }} ================================================ FILE: .github/workflows/validate-yamls.yaml ================================================ name: "[PR Management] Validate Longhorn YAMLs" on: pull_request: types: - opened - edited - synchronize - reopened permissions: contents: read jobs: validate-yamls: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Install yamllint run: | sudo apt-get update sudo apt-get install -y yamllint - name: Create yamllint config run: | cat > .yamllint < [!NOTE] > > To encourage adoption of Longhorn and share your experience with the community, please add your information below. ### Adopter Types - End-User: An organization that runs Longhorn in a production environment. - Integration: An organization offering a product that integrates with Longhorn by providing additional services or functionality. - Vendor: An organization that includes Longhorn as part of its packaged product. - Hosted-Service: An organization that operates Longhorn within its hosted service, offered as a product to customers. ### Longhorn Adopters | Type | Name | Website | Use-Case | |:-|:-|:-|:-| | End-User | Nemlig | https://www.nemlig.com/ | A comprehensive, persistent storage solution on Kubernetes. Part of “Kubernemlig,” Nemlig’s internal, developer- and operations-friendly Platform-as-a-Service (PaaS). | | End-User | Child Rescue Coalition | https://childrescuecoalition.org/ | Kubernetes persistent replicated storage with automated backups. | ================================================ FILE: CHANGELOG/CHANGELOG-1.10.0.md ================================================ # Longhorn v1.10.0 Release Notes Longhorn v1.10.0 is a major release focused on improving stability, performance, and the overall user experience. This version introduces significant enhancements to our core features, including the V2 Data Engine, and streamlines configuration for easier management. The key highlights include improvements to the V2 Data Engine, enhanced resilience, simplified configuration, and better observability. We welcome feedback and contributions to help continuously improve Longhorn. For terminology and context on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). ## Removal ### `longhorn.io/v1beta1` API The `v1beta1` Longhorn API version has been removed. See [GitHub Issue #10249](https://github.com/longhorn/longhorn/issues/10249) for details. ### `replica.status.evictionRequested` Field The deprecated `replica.status.evictionRequested` field has been removed. See [GitHub Issue #7022](https://github.com/longhorn/longhorn/issues/7022) for details. ## Primary Highlights ### New V2 Data Engine Features #### Interrupt Mode Support Interrupt mode has been added to the V2 Data Engine to help reduce CPU usage. This feature is especially beneficial for clusters with idle or low I/O workloads, where conserving CPU resources is more important than minimizing latency. While interrupt mode lowers CPU consumption, it may introduce slightly higher I/O latency compared to polling mode. In addition, the current implementation uses a hybrid approach, which still incurs a minimal, constant CPU load even when interrupts are enabled. > [!NOTE] > **Limitation:** Supports **AIO disks only**. See [Interrupt Mode](https://longhorn.io/docs/1.10.0/v2-data-engine/features/interrupt-mode) and [GitHub Issue#9834](https://github.com/longhorn/longhorn/issues/9834) for details. #### Volume and Snapshot Cloning V2 volumes now support two types of cloning: - **Full-Copy Clone**: Creates a new PVC with a complete, independent copy of the source data, providing full isolation. - **Linked-Clone (Fast/Smart Clone)**: Creates a PVC that shares data blocks with the source volume for near-instant creation. Ideal for temporary workloads, backups, or testing. Linked-clones are lightweight, fast, and reduce storage overhead. See [Volume Clone Support](https://longhorn.io/docs/1.10.0/v2-data-engine/features/volume-clone) and [GitHub Issue#7794](https://github.com/longhorn/longhorn/issues/7794) for details. #### Replica Rebuild QoS Provides Quality of Service (QoS) control for V2 volume replica rebuilds. You can configure bandwidth limits globally or per volume to prevent storage throughput overload on source and destination nodes. See [Replica Rebuild QoS](https://longhorn.io/docs/1.10.0/v2-data-engine/features/replica-rebuild-qos) and [GitHub Issue#10770](https://github.com/longhorn/longhorn/issues/10770) for details. #### Volume Expansion Longhorn now supports volume expansion for V2 Data Engine volumes. You can expand the volume through the UI or by modifying the PVC manifest. See [V2 Volume Expansion](https://longhorn.io/docs/1.10.0/v2-data-engine/features/volume-expansion) and [GitHub Issue#8022](https://github.com/longhorn/longhorn/issues/8022) for details. #### Support for Running Without Hugepages This reduces memory pressure on low-spec nodes and increases deployment flexibility. Performance may be lower compared to running with Hugepages. See [GitHub Issue#7066](https://github.com/longhorn/longhorn/issues/7066) for details. ### New V1 Data Engine Features #### IPv6 Support V1 volumes now support single-stack IPv6 Kubernetes clusters. > **Warning:** Dual-stack Kubernetes clusters and V2 volumes are **not supported** in this release. See [GitHub Issue #2259](https://github.com/longhorn/longhorn/issues/2259) for details. ### Consolidated Global Settings To simplify management, Longhorn settings are now unified across V1 and V2 Data Engines, using a new, more flexible JSON format. - **Single value (applies to all Data Engines)**: Non-JSON string (e.g., `1024`). - **Data-engine-specific**: JSON object (e.g., `{"v1": "value1", "v2": "value2"}`) - **V1-only**: JSON object with v1 key (e.g., `{"v1":"value1"}`). - **V2-only**: JSON object with v2 key (e.g., `{"v2":"value1"}`). See [Longhorn Settings](https://longhorn.io/docs/1.10.0/references/settings) and [GitHub Issue#10926](https://github.com/longhorn/longhorn/issues/10926) for details. ### Pod Scheduling with CSIStorageCapacity Longhorn now supports **CSIStorageCapacity**, allowing Kubernetes to verify node storage before scheduling pods using StorageClasses with **WaitForFirstConsumer**. This reduces scheduling errors and improves reliability. See [GitHub Issue #10685](https://github.com/longhorn/longhorn/issues/10685) for details. ### Configurable Backup Block Size Backup block size can now be configured when creating a volume to optimize performance and efficiency. See [Create Longhorn Volumes](https://longhorn.io/docs/1.10.0/nodes-and-volumes/volumes/create-volumes) and [GitHub Issue#5215](https://github.com/longhorn/longhorn/issues/5215) for details. ### Volume Attachment Summary The UI now shows a summary of attachment tickets on each volume page for improved visibility. See [GitHub Issue #11400](https://github.com/longhorn/longhorn/issues/11400) for details. ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.10.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.10.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.9.x to v1.10.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.10.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ### Highlight - [FEATURE] V2 Volume Supports Cloning [7794](https://github.com/longhorn/longhorn/issues/7794) - @yangchiu @PhanLe1010 - [FEATURE] v2 supports volume expansion [8022](https://github.com/longhorn/longhorn/issues/8022) - @davidcheng0922 @chriscchien - [UI][FEATURE] V2 Volume Supports Cloning [11736](https://github.com/longhorn/longhorn/issues/11736) - @yangchiu @houhoucoop - [FEATURE] V2 volumes support interrupt mode [9834](https://github.com/longhorn/longhorn/issues/9834) - @yangchiu @c3y1huang - [FEATURE] Support v2 volume without hugepage [7066](https://github.com/longhorn/longhorn/issues/7066) - @derekbit @chriscchien - [FEATURE] Configurable Backup Block Size [5215](https://github.com/longhorn/longhorn/issues/5215) - @COLDTURNIP @yangchiu - [UI][FEATURE] Configurable Backup Block Size [11586](https://github.com/longhorn/longhorn/issues/11586) - - [FEATURE] Add QoS support to limit replica rebuilding load [10770](https://github.com/longhorn/longhorn/issues/10770) - @hookak @roger-ryao - [FEATURE] Volume granular setting parity for V2 to match V1 data engine [10926](https://github.com/longhorn/longhorn/issues/10926) - @derekbit @chriscchien - [IMPROVEMENT] Support CSIStorageCapacity in Longhorn CSI driver to enable capacity-aware pod scheduling [10685](https://github.com/longhorn/longhorn/issues/10685) - @bachmanity1 @roger-ryao - [FEATURE] IPV6 for V1 Data Engine [2259](https://github.com/longhorn/longhorn/issues/2259) - @yangchiu @c3y1huang - [FEATURE] Delta Replica Rebuilding using Delta Snapshot: Control and Data Planes [10037](https://github.com/longhorn/longhorn/issues/10037) - @shuo-wu @roger-ryao - [FEATURE] Remove v1beta1 API CRD in Longhorn v1.10 [10249](https://github.com/longhorn/longhorn/issues/10249) - @derekbit @roger-ryao ### Feature - [FEATURE] Add option to restart kubelet through `longhornctl` after huge page update [11241](https://github.com/longhorn/longhorn/issues/11241) - @chriscchien @bachmanity1 - [UI][FEATURE] Configurable Backup Block Size [11351](https://github.com/longhorn/longhorn/issues/11351) - @yangchiu @houhoucoop - [UI][FEATURE] Display a summary of the attachment tickets in an individual volume's overview page [11401](https://github.com/longhorn/longhorn/issues/11401) - @yangchiu @houhoucoop - [UI][FEATURE] Add QoS support to limit replica rebuilding load [11306](https://github.com/longhorn/longhorn/issues/11306) - @davidcheng0922 @houhoucoop @roger-ryao - [UI][FEATURE] Volume granular setting parity for V2 to match V1 data engine [11354](https://github.com/longhorn/longhorn/issues/11354) - @chriscchien @houhoucoop - [FEATURE] Display a summary of the attachment tickets in an individual volume's overview page [11400](https://github.com/longhorn/longhorn/issues/11400) - @yangchiu @davidcheng0922 - [FEATURE] Allow longhorn to restart pods with custom controllers, while the `Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly` feature is enabled [8353](https://github.com/longhorn/longhorn/issues/8353) - @derekbit @roger-ryao - [FEATURE] Standardized way to override container image registry [11064](https://github.com/longhorn/longhorn/issues/11064) - @marcosbc @yangchiu @roger-ryao - [FEATURE] Standardized way to specify image pull secrets [11062](https://github.com/longhorn/longhorn/issues/11062) - @marcosbc @chriscchien ### Improvement - [IMPROVEMENT] Add usage metrics for Longhorn installation variant [11792](https://github.com/longhorn/longhorn/issues/11792) - @derekbit - [IMPROVEMENT] Allow applying different values of snapshot checksum related settings for v1 and v2 data engine [11537](https://github.com/longhorn/longhorn/issues/11537) - @chriscchien @nzhan126 - [IMPROVEMENT] Make `longhornctl` usable in air-gapped environments [11291](https://github.com/longhorn/longhorn/issues/11291) - @chriscchien @bachmanity1 - [IMPROVEMENT] SAST Potential dereference of the null pointer in controller/volume_controller.go in longhorn-manager [11780](https://github.com/longhorn/longhorn/issues/11780) - @c3y1huang - [IMPROVEMENT] Collect Logs from the Host Directory Defined by the Setting `log-path` [11522](https://github.com/longhorn/longhorn/issues/11522) - @c3y1huang @roger-ryao - [IMPROVEMENT] Enhance Offline Rebuilding with Resource Awareness and Retry Backoff [11270](https://github.com/longhorn/longhorn/issues/11270) - @mantissahz @chriscchien - [IMPROVEMENT] Collect mount table, process status and process table in support bundle [8397](https://github.com/longhorn/longhorn/issues/8397) - @mantissahz @chriscchien - [IMPROVEMENT] Volume attachment should automatically exclude nodes with `disable-v2-data-engine="true"` [11695](https://github.com/longhorn/longhorn/issues/11695) - @derekbit @chriscchien - [IMPROVEMENT] Introduce `System Info` Category for Settings [11656](https://github.com/longhorn/longhorn/issues/11656) - @derekbit @roger-ryao - [IMPROVEMENT] RBAC permissions [11345](https://github.com/longhorn/longhorn/issues/11345) - @davidcheng0922 @chriscchien - [IMPROVEMENT] Improve Longhorn Pods Logging Precision to Nanoseconds [11596](https://github.com/longhorn/longhorn/issues/11596) - @derekbit @roger-ryao - [IMPROVEMENT] Update validation logics for v2 data engine [11600](https://github.com/longhorn/longhorn/issues/11600) - @derekbit @chriscchien - [IMPROVEMENT] Improve log messages of longhorn-engine, tgt and liblonghorn for troubleshooting [11545](https://github.com/longhorn/longhorn/issues/11545) - @yangchiu @derekbit - [IMPROVEMENT] rename the backing image manager to reduce the probability of CR name collision [11455](https://github.com/longhorn/longhorn/issues/11455) - @COLDTURNIP @chriscchien - [IMPROVEMENT] Remove outdated prerequisite installation scripts in longhorn/longhorn [11430](https://github.com/longhorn/longhorn/issues/11430) - @yangchiu @roger-ryao @sushant-suse - [UI][IMPROVEMENT] Add UI Warning for Force-Detach Actions to Prevent Out-of-Sync Kubernetes and Longhorn VolumeAttachments [9944](https://github.com/longhorn/longhorn/issues/9944) - @yangchiu @houhoucoop - [IMPROVEMENT] Add `node-selector` option to `longhornctl` to select nodes on which to run DaemonSet [11213](https://github.com/longhorn/longhorn/issues/11213) - @yangchiu @bachmanity1 - [IMPROVEMENT] Improve volume `Scheduled` condition message [11460](https://github.com/longhorn/longhorn/issues/11460) - @yangchiu @derekbit @chriscchien - [IMPROVEMENT] Launching a new mechanism to collect instance manager logs [5948](https://github.com/longhorn/longhorn/issues/5948) - @yangchiu @derekbit - [IMPROVEMENT] adjust the hardcoded timeout limitation for backing image downloading [11309](https://github.com/longhorn/longhorn/issues/11309) - @COLDTURNIP @roger-ryao - [IMPROVEMENT] Make liveness probe parameters of instance-manager pod configurable [10788](https://github.com/longhorn/longhorn/issues/10788) - @yangchiu @derekbit - [IMPROVEMENT] Enhance menu descriptions for Longhorn CLI [8998](https://github.com/longhorn/longhorn/issues/8998) - @roger-ryao @sushant-suse - [IMPROVEMENT] Improve longhorn-engine controller log messages [11507](https://github.com/longhorn/longhorn/issues/11507) - @derekbit @chriscchien - [IMPROVEMENT] Add a comment to explain what `isSettingDataEngineSynced` does in the instance manager controller. [11321](https://github.com/longhorn/longhorn/issues/11321) - @mantissahz - [IMPROVEMENT] Flooding and misleading log message `Deleting orphans on evicted node ...` [11500](https://github.com/longhorn/longhorn/issues/11500) - @yangchiu @derekbit - [IMPROVEMENT] Reject `volume.spec.replicaRebuildingBandwidthLimit` update for V1 Data Engine [11497](https://github.com/longhorn/longhorn/issues/11497) - @derekbit @roger-ryao - [IMPROVEMENT] Detach an offline rebuilding volume if rebuilding can not start [11274](https://github.com/longhorn/longhorn/issues/11274) - @mantissahz - [IMPROVEMENT] backing image handle node disk deleting events [10983](https://github.com/longhorn/longhorn/issues/10983) - @COLDTURNIP @chriscchien - [IMPROVEMENT] Rename `RebuildingMbytesPerSecond` to `ReplicaRebuildBandwidthLimit` [11403](https://github.com/longhorn/longhorn/issues/11403) - @derekbit @roger-ryao - [IMPROVEMENT] Make the sync agent profilable [11386](https://github.com/longhorn/longhorn/issues/11386) - @COLDTURNIP @yangchiu - [IMPROVEMENT] Add performance metrics for Longhorn disk I/O [11223](https://github.com/longhorn/longhorn/issues/11223) - @hookak @DamiaSan - [IMPROVEMENT] Make CLI preflight check non-blocking for subsequent checkups [9877](https://github.com/longhorn/longhorn/issues/9877) - @davidcheng0922 @DamiaSan - [IMPROVEMENT] Add namespace argument/parameter to cli pre-flight check [9749](https://github.com/longhorn/longhorn/issues/9749) - @davidcheng0922 @DamiaSan - [IMPROVEMENT] `Orphaned Data` should not be placed under Settings [10383](https://github.com/longhorn/longhorn/issues/10383) - @houhoucoop @DamiaSan @sushant-suse - [IMPROVEMENT] Upgrade Node v20 in longhorn-ui [11315](https://github.com/longhorn/longhorn/issues/11315) - @chriscchien @houhoucoop - [IMPROVEMENT] useful error message from /v1/backuptargets is not displayed in UI [10428](https://github.com/longhorn/longhorn/issues/10428) - @houhoucoop @DamiaSan - [IMPROVEMENT] Check if the backup target is available before creating a backup, backup backing image, and system backup [10085](https://github.com/longhorn/longhorn/issues/10085) - @yangchiu @nzhan126 - [IMPROVEMENT] Backoff Retry Interval for Instance Manager Pod Re-creation in Resource Constraint Scenarios [10263](https://github.com/longhorn/longhorn/issues/10263) - @yangchiu @bachmanity1 - [IMPROVEMENT] record the detail while webhook rejecting migration attachment tickets [11150](https://github.com/longhorn/longhorn/issues/11150) - @COLDTURNIP @roger-ryao - [IMPROVEMENT] Handle credential secret containing mixed invalid conditions [8537](https://github.com/longhorn/longhorn/issues/8537) - @yangchiu @nzhan126 - [IMPROVEMENT] Add the possibility of setting floating point values for `guaranteed-instance-manager-cpu` and `node.spec.instanceManagerCPURequest` [11179](https://github.com/longhorn/longhorn/issues/11179) - @yangchiu @gigabyte132 - [IMPROVEMENT] Remove the Patch `preserveUnknownFields: false` for CRDs [11263](https://github.com/longhorn/longhorn/issues/11263) - @derekbit @chriscchien - [IMPROVEMENT] Schedule at least one replica locally when locality is `best-effort` [11007](https://github.com/longhorn/longhorn/issues/11007) - @chriscchien @bachmanity1 - [IMPROVEMENT] Improve the disk space un-schedulable condition message [10436](https://github.com/longhorn/longhorn/issues/10436) - @yangchiu @davidcheng0922 - [IMPROVEMENT] Improve the condition message of engine image check [9845](https://github.com/longhorn/longhorn/issues/9845) - @derekbit @chriscchien - [IMPROVEMENT] Improve the logging when detecting multiple backup volumes of the same volume on the same backup target [11152](https://github.com/longhorn/longhorn/issues/11152) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Implement Documentation Validation for `longhorn/cli` [11229](https://github.com/longhorn/longhorn/issues/11229) - @derekbit - [IMPROVEMENT] Move validation from each resource deletion to validation webhook [5156](https://github.com/longhorn/longhorn/issues/5156) - @derekbit @roger-ryao - [IMPROVEMENT] Validate node.longhorn.io resource spec fields [11079](https://github.com/longhorn/longhorn/issues/11079) - @Felipalds @chriscchien - [IMPROVEMENT] add support for custom annotations in the UI service on Longhorn Helm Chart [11031](https://github.com/longhorn/longhorn/issues/11031) - @josimar-silva @roger-ryao - [IMPROVEMENT] Adding retry logic for longhorn-csi-plugin when it trying to contact the longhorn-manager pods [9482](https://github.com/longhorn/longhorn/issues/9482) - @PhanLe1010 @roger-ryao ### Bug - [BUG] failed to load DataEngineSpecific boolean setting from configmap [11810](https://github.com/longhorn/longhorn/issues/11810) - @COLDTURNIP @roger-ryao - [BUG] V2 stop working - connectNVMfBdev() -> "code": -95,"message": "Operation not supported" (1.10.0-rc2) [11761](https://github.com/longhorn/longhorn/issues/11761) - @yangchiu @c3y1huang - [BUG] [UI] Inconsistent Default Value for Data Engine in Clone Volume [11802](https://github.com/longhorn/longhorn/issues/11802) - @houhoucoop @roger-ryao - [BUG] System backup could get stuck in `CreatingVolumeBackups` if some nodes are labeled with `disable-v2-data-engine=true` [11774](https://github.com/longhorn/longhorn/issues/11774) - @mantissahz @roger-ryao - [BUG] Potential Data Corruption During Volume Resizing When Created from Snapshot [11484](https://github.com/longhorn/longhorn/issues/11484) - @yangchiu @PhanLe1010 - [BUG] Block disk may never become `Schedulable` after re-adding [11760](https://github.com/longhorn/longhorn/issues/11760) - @derekbit @chriscchien - [BUG] v2 volume could get stuck in `Detaching/Faulted` state after nodes reboot [10112](https://github.com/longhorn/longhorn/issues/10112) - @yangchiu @shuo-wu - [BUG] Fail to dynamically provision a v2 volume with a backing image if the backing image doesn't exist before PVC creation [11762](https://github.com/longhorn/longhorn/issues/11762) - @COLDTURNIP @yangchiu - [BUG] v2 DR volume faulted after origin volume expand and backuped [11767](https://github.com/longhorn/longhorn/issues/11767) - @davidcheng0922 @roger-ryao - [BUG] longhorn manager crash in installation [11743](https://github.com/longhorn/longhorn/issues/11743) - @derekbit @chriscchien - [BUG] Unable to sync existing backups from a remote backup store [11758](https://github.com/longhorn/longhorn/issues/11758) - @yangchiu @mantissahz - [BUG] Longhorn pvcs are in pending state. [11654](https://github.com/longhorn/longhorn/issues/11654) - @yangchiu @derekbit - [BUG] Volume becomes faulted when its replica node disks run out of space during a write operation [10718](https://github.com/longhorn/longhorn/issues/10718) - @yangchiu @mantissahz - [BUG] [v1.10.0-rc1] `longhornctl trim volume` command hangs [11704](https://github.com/longhorn/longhorn/issues/11704) - @davidcheng0922 @chriscchien - [BUG] longhornctl preflight install should load and check iscsi_tcp kernel module. [11706](https://github.com/longhorn/longhorn/issues/11706) - @mantissahz @chriscchien - [BUG] spdk_tgt crash after replica rebuilding due to bdev_channel_destroy_resource() assert failure [11109](https://github.com/longhorn/longhorn/issues/11109) - @hookak @chriscchien - [BUG] Unable to set replica affinity when creating a v2 volume (`test_soft_anti_affinity_scheduling_volume_enable`) [11642](https://github.com/longhorn/longhorn/issues/11642) - @yangchiu @derekbit - [BUG] BackupBackingImage may be created from an unready BackingImageManager [11675](https://github.com/longhorn/longhorn/issues/11675) - @WebberHuang1118 @roger-ryao - [BUG] Creating a 2 Gi volume with a 200 Mi backing image is rejected with “volume size should be larger than the backing image size” [11362](https://github.com/longhorn/longhorn/issues/11362) - @COLDTURNIP @yangchiu - [BUG] Replica auto balance disk in pressure fails on v2 volumes [10551](https://github.com/longhorn/longhorn/issues/10551) - @yangchiu @hookak - [BUG] Backup stuck when ownerID is assigned to a node with node.longhorn.io/disable-v2-data-engine: "true" [11619](https://github.com/longhorn/longhorn/issues/11619) - @davidcheng0922 @roger-ryao - [BUG] Engine process continues running after rapid volume detachment [11605](https://github.com/longhorn/longhorn/issues/11605) - @COLDTURNIP @yangchiu - [BUG] remaining unknown OS condition in node CR [11612](https://github.com/longhorn/longhorn/issues/11612) - @COLDTURNIP @roger-ryao - [BUG] Longhorn Manager continues to send replica deletion requests to the Instance Manager for the v2 volume indefinitely [11553](https://github.com/longhorn/longhorn/issues/11553) - @yangchiu @shuo-wu - [BUG] Unable to disable v2-data-engine even though there is no v2 volumes, backing images or orphaned data [11330](https://github.com/longhorn/longhorn/issues/11330) - @shuo-wu @roger-ryao - [BUG] longhorn-manager repeatedly emits `No instance manager for node xxx for update instance state of orphan instance orphan-xxx..` [11597](https://github.com/longhorn/longhorn/issues/11597) - @COLDTURNIP @chriscchien - [BUG] Volumes fails to remount when they go read-only [8572](https://github.com/longhorn/longhorn/issues/8572) - @derekbit @chriscchien - [BUG] Dangling Volume State When Live Migration Terminates Unexpectedly [11479](https://github.com/longhorn/longhorn/issues/11479) - @PhanLe1010 @chriscchien - [BUG] S3 Backup target reverts randomly to previous value [9581](https://github.com/longhorn/longhorn/issues/9581) - @yangchiu @mantissahz - [BUG] Longhornctl / CLI - no configuration has been provided, try setting KUBERNETES_MASTER environment variable [10094](https://github.com/longhorn/longhorn/issues/10094) - @davidcheng0922 @chriscchien - [BUG] longhorn-images.txt specifies CSI component repo tags not found [11575](https://github.com/longhorn/longhorn/issues/11575) - @yangchiu @derekbit - [BUG] DR volume's backup block size should be set from the latest backup [11580](https://github.com/longhorn/longhorn/issues/11580) - @COLDTURNIP @yangchiu - [BUG] longhornctl --enable-spdk doesn't support arm64 [11551](https://github.com/longhorn/longhorn/issues/11551) - @yangchiu @davidcheng0922 - [BUG] extra invalid BackupVolumeCR may be created during cluster split-brain [11154](https://github.com/longhorn/longhorn/issues/11154) - @mantissahz @roger-ryao - [BUG] system backup error [11232](https://github.com/longhorn/longhorn/issues/11232) - @c3y1huang @roger-ryao - [BUG] Can not create backup using `Create Backup` icon in UI [11451](https://github.com/longhorn/longhorn/issues/11451) - @yangchiu @mantissahz @houhoucoop - [BUG] Unable to create backup for old snapshots on v2 volumes: failed to find snapshot lvol range [11461](https://github.com/longhorn/longhorn/issues/11461) - @c3y1huang @chriscchien - [BUG] Uninstall fail because find backuptargets remaining [11486](https://github.com/longhorn/longhorn/issues/11486) - @COLDTURNIP @yangchiu - [BUG] Setting v2 data engine can be enabled without fulfilling the hugepage requirement, causing error v2 instance manager CR dangling in the system [11519](https://github.com/longhorn/longhorn/issues/11519) - @yangchiu @derekbit - [BUG] Unable to setup backup target in storage network environment: cannot find a running instance manager for node [11478](https://github.com/longhorn/longhorn/issues/11478) - @yangchiu @derekbit - [BUG] Volume migration negative test cases fail on v2 volumes [10800](https://github.com/longhorn/longhorn/issues/10800) - @shuo-wu @chriscchien - [BUG] V2 volume fails to cleanup error replica and rebuild new one - test_data_locality_basic [10335](https://github.com/longhorn/longhorn/issues/10335) - @shuo-wu @chriscchien - [BUG] Issue auto detecting nvme drive on talos cluster (vfio-pci driver instead of expected vfio_pci) [11127](https://github.com/longhorn/longhorn/issues/11127) - @Hugome @roger-ryao - [BUG] Test case `test_running_volume_with_scheduling_failure` failed due to unexpected new replica created [11512](https://github.com/longhorn/longhorn/issues/11512) - @derekbit @chriscchien - [BUG] Regression test cases failed due to unable to clean up dummy backups [11487](https://github.com/longhorn/longhorn/issues/11487) - @COLDTURNIP @yangchiu - [BUG][v1.9.0-rc1] Unexpected orphaned data are created after v2 instance managers deleted [10829](https://github.com/longhorn/longhorn/issues/10829) - @yangchiu @derekbit - [BUG] v2 Engine loops in detaching and attaching state after rebuilding [10396](https://github.com/longhorn/longhorn/issues/10396) - @shuo-wu @roger-ryao - [BUG] `test_basic.py::test_backup_status_for_unavailable_replicas` is failed [11416](https://github.com/longhorn/longhorn/issues/11416) - @derekbit @roger-ryao - [BUG] Build fails due to outdated SLES repo [11481](https://github.com/longhorn/longhorn/issues/11481) - @PhanLe1010 - [BUG] Unable to set up S3 backup target if backups already exist [11337](https://github.com/longhorn/longhorn/issues/11337) - @mantissahz @chriscchien - [BUG][UI] `Snapshots and Backups` graph is stuck loading and console shows error messages [10529](https://github.com/longhorn/longhorn/issues/10529) - @yangchiu @davidcheng0922 - [BUG] The Backup YAML example in the Longhorn doc does not work [11216](https://github.com/longhorn/longhorn/issues/11216) - @roger-ryao @nzhan126 - [BUG] v2 volume workload IO could get `Bad message` error after network disconnect [10113](https://github.com/longhorn/longhorn/issues/10113) - @yangchiu @shuo-wu - [BUG] Test case `Test Replica Auto Balance Node Least Effort` failed on v2 volume [10977](https://github.com/longhorn/longhorn/issues/10977) - @yangchiu @c3y1huang - [BUG] IsJSONRPCRespErrorNoSuchDevice fails on wrapped errors for ublk client [11361](https://github.com/longhorn/longhorn/issues/11361) - @yangchiu @davidcheng0922 - [BUG] longhorn-manager is crashed due to `SIGSEGV: segmentation violation` [11420](https://github.com/longhorn/longhorn/issues/11420) - @derekbit @chriscchien - [BUG] Test Case `test_replica_auto_balance_node_least_effort` Is Sometimes Failed [11388](https://github.com/longhorn/longhorn/issues/11388) - @derekbit @chriscchien - [BUG] volume gets stuck at detaching/faulted state for spdk v2 engine intermittently [10724](https://github.com/longhorn/longhorn/issues/10724) - @DamiaSan - [BUG] Typo in configuration parameter: "offlineRelicaRebuilding" should be "offlineReplicaRebuilding" [11380](https://github.com/longhorn/longhorn/issues/11380) - @yangchiu @in-jun - [BUG][DOC] OpenShift documentation [11174](https://github.com/longhorn/longhorn/issues/11174) - @yangchiu @mlacko64 - [BUG] In single node Harvester, endless "unable to schedule replica" is logged in longhorn-manager [3708](https://github.com/longhorn/longhorn/issues/3708) - @derekbit @chriscchien - [BUG] Uninstallation fail due to deleting the running Longhorn node is not allowed [11131](https://github.com/longhorn/longhorn/issues/11131) - @COLDTURNIP @roger-ryao - [BUG] Engine v2 I/O Blocked Over 1-2 Minutes After Instance Manager Pod Deletion [10167](https://github.com/longhorn/longhorn/issues/10167) - @c3y1huang @chriscchien - [BUG] Regression test cases failed: expecting volume to be detached but it's attached [11273](https://github.com/longhorn/longhorn/issues/11273) - @yangchiu @mantissahz - [BUG] Volume expansion fails with "unsupported disk encryption format ext4" [11120](https://github.com/longhorn/longhorn/issues/11120) - @COLDTURNIP @mantissahz @roger-ryao - [BUG] longhorn-spdk-engine rebuilding unit tests may get stuck for 2 minutes [11099](https://github.com/longhorn/longhorn/issues/11099) - @shuo-wu @roger-ryao - [BUG] v2 volume could get stuck in `detaching/detached` loop when `Migration Confirmation After Migration Node Down` [10157](https://github.com/longhorn/longhorn/issues/10157) - @yangchiu @PhanLe1010 - [BUG] Longhorn will not reuse the failed v2 replicas when a race condition is triggered after the instance manager pod restart [11188](https://github.com/longhorn/longhorn/issues/11188) - @shuo-wu @roger-ryao - [BUG] Auto-generated CLI document overwrite by make [11219](https://github.com/longhorn/longhorn/issues/11219) - @bachmanity1 - [BUG] Incorrect value of `remove-snapshots-during-filesystem-trim` in longhorn chart/values.yaml [11264](https://github.com/longhorn/longhorn/issues/11264) - @derekbit @chriscchien - [BUG][v1.9.0-rc1] v2 volumes don't reuse failed replicas as expected after a node goes down [10828](https://github.com/longhorn/longhorn/issues/10828) - @yangchiu @shuo-wu - [BUG] CSI Plugin restart triggers unintended restart of migratable RWX volume workloads [11158](https://github.com/longhorn/longhorn/issues/11158) - @c3y1huang @roger-ryao - [BUG] in the browser UI: Volume -> Clone Volume results in the broken browser page [11165](https://github.com/longhorn/longhorn/issues/11165) - @houhoucoop @roger-ryao - [BUG] "mkfsParams" in StorageClass are not passed to share-manager for filesystem formatting [11107](https://github.com/longhorn/longhorn/issues/11107) - @Florianisme @roger-ryao - [BUG] Test case `test_engine_image_not_fully_deployed_perform_volume_operations` failed: unable to detach a volume [10874](https://github.com/longhorn/longhorn/issues/10874) - @mantissahz @chriscchien - [BUG] Creating support-bundle panic NPE [11169](https://github.com/longhorn/longhorn/issues/11169) - @c3y1huang @roger-ryao - [BUG] Unable to Build Longhorn-Share-Manager Image Due to CMAKE Compatibility [11159](https://github.com/longhorn/longhorn/issues/11159) - @derekbit @roger-ryao - [BUG] Uninstallation fail due to deleting the default engine image is not allowed [11130](https://github.com/longhorn/longhorn/issues/11130) - @COLDTURNIP @chriscchien - [BUG] v2 volume gets stuck in degraded state and continuously rebuilds/deletes replicas after a kubelet restart [10107](https://github.com/longhorn/longhorn/issues/10107) - @shuo-wu - [BUG] SPDK API bdev_lvol_detach_parent does not work as expected [11046](https://github.com/longhorn/longhorn/issues/11046) - @DamiaSan @roger-ryao - [BUG] Recurring jobs fail when assigned to default group [11016](https://github.com/longhorn/longhorn/issues/11016) - @c3y1huang @chriscchien - [BUG] v2 volume data checksum mismatch after replica rebuilding [10118](https://github.com/longhorn/longhorn/issues/10118) - @yangchiu @shuo-wu - [BUG] Most of regression test cases are failing due to unable to update settings [11042](https://github.com/longhorn/longhorn/issues/11042) - @yangchiu @mantissahz - [BUG] unable to clean up the backing image volume replica after node eviction [11053](https://github.com/longhorn/longhorn/issues/11053) - @COLDTURNIP @roger-ryao - [BUG] backing image volume replica NPE crash during evicting node [11034](https://github.com/longhorn/longhorn/issues/11034) - @COLDTURNIP @chriscchien - [BUG] A degraded DR volume remains in standby and attached after activation. [2107](https://github.com/longhorn/longhorn/issues/2107) - @roger-ryao - [BUG] DR volume gets stuck if there is only a rebuilding replica running [2753](https://github.com/longhorn/longhorn/issues/2753) - @c3y1huang @roger-ryao ### Stability - [DOC] Document Volume Stability Risks Caused by I/O Latency on HDDs [11240](https://github.com/longhorn/longhorn/issues/11240) - @chriscchien @sushant-suse ### Misc - [DOC] Add Information on Replica Failure Tolerance [11526](https://github.com/longhorn/longhorn/issues/11526) - @roger-ryao @sushant-suse - [TASK] KB for backup store lock conflict error message [11293](https://github.com/longhorn/longhorn/issues/11293) - @yangchiu @pratikjagrut - [DOC] Document Replica Rebuilding Mechanisms and Their Limitations [11119](https://github.com/longhorn/longhorn/issues/11119) - @mantissahz @chriscchien - [DOC] V2 Engine usage contradiction [11409](https://github.com/longhorn/longhorn/issues/11409) - @shuo-wu @roger-ryao - [DOC] Talos new volumes strategy [11015](https://github.com/longhorn/longhorn/issues/11015) - @yangchiu @DrummyFloyd - [TASK] [pytest] automatically add -m v2_volume_test if RUN_V2_TEST enabled [11376](https://github.com/longhorn/longhorn/issues/11376) - @chriscchien - Revise the document about node space [3021](https://github.com/longhorn/longhorn/issues/3021) - @derekbit @chriscchien - [DOC] Troubleshooting KB for Mount Failure with XFS Filesystem [11214](https://github.com/longhorn/longhorn/issues/11214) - @derekbit @roger-ryao - [DOC] Explain Longhorn VolumeAttachment operation and behavior [11142](https://github.com/longhorn/longhorn/issues/11142) - @derekbit @roger-ryao - [DOC] Update Broken Links on Website [11288](https://github.com/longhorn/longhorn/issues/11288) - @yangchiu @sushant-suse - [DOC] Clarify privateRegistry.createSecret and registrySecret usage in chart README [11251](https://github.com/longhorn/longhorn/issues/11251) - @chriscchien - [DOC] KB: failed to complete volume migration during VM upgrade [11149](https://github.com/longhorn/longhorn/issues/11149) - @COLDTURNIP @chriscchien - [DOC] Elaborate how to enable DR volume using kubectl [10958](https://github.com/longhorn/longhorn/issues/10958) - @derekbit @chriscchien - [DOC] Remove `defaultSettings.registrySecret` reference from air gap installation guide [11237](https://github.com/longhorn/longhorn/issues/11237) - @chriscchien - [TASK] Remove deprecated replica.status.evictionRequested field [7022](https://github.com/longhorn/longhorn/issues/7022) - @yangchiu @derekbit @roger-ryao - [TASK] Create longhorn/spdk longhorn-v25.05 branch [11048](https://github.com/longhorn/longhorn/issues/11048) - @derekbit @chriscchien - [TASK] Create a Dedicated Repository for libqcow to Improve Maintainability and Build Management [10988](https://github.com/longhorn/longhorn/issues/10988) - @derekbit @chriscchien - [DOC] Update examples using deprecated crd version v1beta1 to v1beta2 on 1.9.0 [11019](https://github.com/longhorn/longhorn/issues/11019) - @falmar @roger-ryao - [TASK] POC for ui-extension migration [10516](https://github.com/longhorn/longhorn/issues/10516) - @houhoucoop - [TASK] Ensure support-bundle-kit builds use vendored dependencies [11106](https://github.com/longhorn/longhorn/issues/11106) - @yangchiu @c3y1huang - [DOC] Fix the broken links present in documentation [11028](https://github.com/longhorn/longhorn/issues/11028) - @chriscchien @sushant-suse - [DOC] Update website front page to include community meeting links [10890](https://github.com/longhorn/longhorn/issues/10890) - @yangchiu @divya-mohan0209 @sushant-suse - [REFACTOR] Use go pkg for system operation instead of relying on external system call via shell command [5193](https://github.com/longhorn/longhorn/issues/5193) - @c3y1huang ## New Contributors - @Felipalds - @Florianisme - @Hugome - @divya-mohan0209 - @falmar - @gigabyte132 - @in-jun - @josimar-silva - @mlacko64 - @pratikjagrut ## Contributors - @COLDTURNIP - @DamiaSan - @DrummyFloyd - @PhanLe1010 - @WebberHuang1118 - @bachmanity1 - @c3y1huang - @chriscchien - @davidcheng0922 - @derekbit - @hookak - @houhoucoop - @innobead - @mantissahz - @marcosbc - @nzhan126 - @roger-ryao - @shuo-wu - @sushant-suse - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.10.1.md ================================================ # Longhorn v1.10.1 Release Notes Longhorn 1.10.1 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. We welcome feedback and contributions to help continuously improve Longhorn. For terminology and context on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). ## Important Fixes This release includes several critical stability and performance improvements: ### Goroutine Leak in Instance Manager (V2 Data Engine) Fixed a goroutine leak in the instance manager when using the V2 data engine. This issue could lead to increased memory usage and potential stability problems over time. For more details, see [Issue #11962](https://github.com/longhorn/longhorn/issues/11962). ### V2 Volume Attachment Failure in Interrupt Mode Fixed an issue where V2 volumes using interrupt mode with NVMe disks could fail to complete the attachment process, causing volumes to remain stuck in the attaching state indefinitely. In Longhorn v1.10.0, interrupt mode supports only **AIO disks**. Interrupt mode for **NVMe disks** is supported starting in v1.10.1. For more details, see [Issue #11816](https://github.com/longhorn/longhorn/issues/11816). ### UI Deployment Failure on IPv4-Only Nodes Fixed a bug introduced in v1.10.0 where the Longhorn UI failed to deploy on nodes with only IPv4 enabled. The UI now correctly supports IPv4-only configurations without requiring IPv6. For more details, see [Issue #11875](https://github.com/longhorn/longhorn/issues/11875). ### Share Manager Excessive Memory Usage Fixed excessive memory consumption in the share manager for RWX (ReadWriteMany) volumes. The component now maintains stable memory usage under normal operation. For more details, see [Issue #12043](https://github.com/longhorn/longhorn/issues/12043). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.10.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.10.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.9.x to v1.10.1.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.10.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.10.1][IMPROVEMENT] The `auto-delete-pod-when-volume-detached-unexpectedly` should only focus on the kubernetes builtin workload. [12125](https://github.com/longhorn/longhorn/issues/12125) - @derekbit @chriscchien - [BACKPORT][v1.10.1][IMPROVEMENT] `CSIStorageCapacity` objects must show schedulable (allocatable) capacity [12036](https://github.com/longhorn/longhorn/issues/12036) - @chriscchien @bachmanity1 - [BACKPORT][v1.10.1][IMPROVEMENT] improve error logging for failed mounting during node publish volume [12033](https://github.com/longhorn/longhorn/issues/12033) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.10.1][IMPROVEMENT] Improve Helm Chart defaultSettings handling with automatic quoting and multi-type support [12020](https://github.com/longhorn/longhorn/issues/12020) - @derekbit @chriscchien - [BACKPORT][v1.10.1][IMPROVEMENT] Avoid repeat engine restart when there are replica unavailable during migration [11945](https://github.com/longhorn/longhorn/issues/11945) - @yangchiu @shuo-wu - [BACKPORT][v1.10.1][IMPROVEMENT] Adjust maximum of GuaranteedInstanceManagerCPU to a big value [11968](https://github.com/longhorn/longhorn/issues/11968) - @mantissahz - [BACKPORT][v1.10.1][IMPROVEMENT] Add usage metrics for Longhorn installation variant [11795](https://github.com/longhorn/longhorn/issues/11795) - @derekbit ### Bug - [BACKPORT][v1.10.1][BUG] Backup target metric is broken [12089](https://github.com/longhorn/longhorn/issues/12089) - @mantissahz @roger-ryao - [BACKPORT][v1.10.1][BUG] Backing image download gets stuck after network disconnection [12094](https://github.com/longhorn/longhorn/issues/12094) - @COLDTURNIP @chriscchien - [BACKPORT][v1.10.1][BUG] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 at longhorn-engine/pkg/controller/control.go:218 +0x2de [12088](https://github.com/longhorn/longhorn/issues/12088) - @roger-ryao - [BACKPORT][v1.10.1][BUG] Unable to complete uninstallation due to the remaining backuptarget [11964](https://github.com/longhorn/longhorn/issues/11964) - @mantissahz @roger-ryao - [BACKPORT][v1.10.1][BUG] share-manager excessive memory usage [12043](https://github.com/longhorn/longhorn/issues/12043) - @derekbit @chriscchien - [BACKPORT][v1.10.1][BUG] NVME disk not found in v2 data engine (failed to find device for BDF) [12029](https://github.com/longhorn/longhorn/issues/12029) - @derekbit @roger-ryao - [BACKPORT][v1.10.1][BUG] NPE error during recurring job execution [11926](https://github.com/longhorn/longhorn/issues/11926) - @yangchiu @shuo-wu - [BACKPORT][v1.10.1][BUG] v2 volume creation failed on talos nodes [12026](https://github.com/longhorn/longhorn/issues/12026) - @c3y1huang @chriscchien - [BACKPORT][v1.10.1][BUG] mounting error is not properly hanedled during CSI node publish volume [12008](https://github.com/longhorn/longhorn/issues/12008) - @COLDTURNIP - [BACKPORT][v1.10.1][BUG] Adding multiple disks to the same node concurrently may occasionally fail [12018](https://github.com/longhorn/longhorn/issues/12018) - @davidcheng0922 @roger-ryao - [BUG] upgrading from 1.9.1 to 1.10.0 fails due to old resources still being in v1beta1 [11886](https://github.com/longhorn/longhorn/issues/11886) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.10.1][BUG] DR volume gets stuck in `unknown` state if engine image is deleted from the attached node [11998](https://github.com/longhorn/longhorn/issues/11998) - @yangchiu @shuo-wu - [BACKPORT][v1.10.1][BUG] Volume gets stuck in `attaching` state if engine image image is not deployed on one of nodes [11996](https://github.com/longhorn/longhorn/issues/11996) - @yangchiu @shuo-wu - [BACKPORT][v1.10.1][BUG] Unable to re-add block-type disks by BDF after re-enable v2 data engine [12000](https://github.com/longhorn/longhorn/issues/12000) - @yangchiu @davidcheng0922 - [BACKPORT][v1.10.1][BUG] `test_system_backup_and_restore` test case failed on master-head [12005](https://github.com/longhorn/longhorn/issues/12005) - @derekbit @chriscchien - [BACKPORT][v1.10.1][BUG] Fix SPDK v25.05 CVE issue [11970](https://github.com/longhorn/longhorn/issues/11970) - @derekbit @roger-ryao - [BACKPORT][v1.10.1][BUG] V2 volume stuck in volume attachment (V2 interrupt mode) [11976](https://github.com/longhorn/longhorn/issues/11976) - @c3y1huang @chriscchien - [BACKPORT][v1.10.1][BUG] RWX volume causes process uninterruptible sleep [11958](https://github.com/longhorn/longhorn/issues/11958) - @COLDTURNIP @chriscchien - [BACKPORT][v1.10.1][BUG] longhorn-manager fails to start after upgrading from 1.9.2 to 1.10.0 [11865](https://github.com/longhorn/longhorn/issues/11865) - @derekbit @roger-ryao - [BACKPORT][v1.10.1][BUG] Block disk deletion fails without error message [11954](https://github.com/longhorn/longhorn/issues/11954) - @davidcheng0922 @roger-ryao - [BACKPORT][v1.10.1][BUG] Goroutine leak in instance-manager when using v2 data engine [11962](https://github.com/longhorn/longhorn/issues/11962) - @PhanLe1010 @chriscchien - [BACKPORT][v1.10.1][BUG] invalid memory address or nil pointer dereference [11942](https://github.com/longhorn/longhorn/issues/11942) - @bachmanity1 @roger-ryao - [BACKPORT][v1.10.1][BUG] csi-provisioner silently fails to create CSIStorageCapacity if dataEngine parameter is missing [11918](https://github.com/longhorn/longhorn/issues/11918) - @yangchiu @bachmanity1 - [BACKPORT][v1.10.1][BUG] longhorn-engine's UI panics [11901](https://github.com/longhorn/longhorn/issues/11901) - @derekbit @chriscchien - [BACKPORT][v1.10.1][BUG] Volume is unable to upgrade if the number of active replicas is larger than `volumme.spec.numberOfReplicas` [11895](https://github.com/longhorn/longhorn/issues/11895) - @yangchiu @derekbit - [BACKPORT][v1.10.1][BUG] UI fails to deploy when only IPv4 is enabled on nodes with v1.10.0 version [11875](https://github.com/longhorn/longhorn/issues/11875) - @yangchiu @c3y1huang - [BACKPORT][v1.10.1][BUG] Unable to detach a v2 volume after labeling `disable-v2-data-engine=true` [11801](https://github.com/longhorn/longhorn/issues/11801) - @mantissahz ### Misc - [BACKPORT][v1.10.1][REFACTOR] SAST checks for UI component [11992](https://github.com/longhorn/longhorn/issues/11992) - @chriscchien - [HOTFIX] Create hotfixed image for longhorn-manager:v1.10.0 [11951](https://github.com/longhorn/longhorn/issues/11951) - @c3y1huang @roger-ryao ## Contributors - @COLDTURNIP - @PhanLe1010 - @bachmanity1 - @c3y1huang - @chriscchien - @davidcheng0922 - @derekbit - @forbesguthrie - @innobea - @mantissahz - @rebeccazzzz - @roger-ryao - @sushant-suse - @shuo-wu - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.10.2.md ================================================ # Longhorn v1.10.2 Release Notes Longhorn 1.10.2 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. We welcome feedback and contributions to help continuously improve Longhorn. For terminology and context on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). ## Important Fixes This release includes several critical stability fixes. ### RWX Volume Unavailable After Node Drain Fixed a race condition where **ReadWriteMany (RWX) volumes** could remain in the *attaching* state after node drains, causing workloads to become unavailable. For more details, see [Issue #12231](https://github.com/longhorn/longhorn/issues/12231). ### Encrypted Volume Cannot Be Expanded Online Fixed an issue where online expansion of encrypted volumes did not propagate the new size to the dm-crypt device. For more details, see [Issue #12368](https://github.com/longhorn/longhorn/issues/12368). ### Cloned Volume Cannot Be Attached to Workload Fixed a bug where cloned volumes could fail to reach a healthy state, preventing attachment to workloads. For more details, see [Issue #12208](https://github.com/longhorn/longhorn/issues/12208). ### Block Mode Volume Migration Stuck Fixed a regression in block-mode volume migrations where newly created replicas could incorrectly inherit the `lastFailedAt` timestamp from source replicas, causing repeated deletion and blocking migration completion. For more details, see [Issue #12312](https://github.com/longhorn/longhorn/issues/12312). ### Replica Auto Balance Disk Pressure Threshold Stalled Fixed an issue where replica auto-balance under disk pressure could be blocked if stopped volumes were present on the disk. For more details, see [Issue #12334](https://github.com/longhorn/longhorn/issues/12334). ### Replicas Accumulate During Engine Upgrade Fixed a bug where temporary replicas could accumulate during engine upgrade. High etcd latency could cause new replicas to fail verification, leading to accumulation over multiple reconciliation cycles. For more details, see [Issue #12115](https://github.com/longhorn/longhorn/issues/12115). ### Potential Client Connection and Context Leak Fixed potential context leaks in the instance manager client and backing image manager client, improving stability and preventing resource exhaustion. For more details, see [Issue #12200](https://github.com/longhorn/longhorn/issues/12200) and [Issue #12195](https://github.com/longhorn/longhorn/issues/12195). ### Replica Node Level Soft Anti-Affinity Ignored Fixed a bug of replica scheduling loop where replicas could be scheduled onto nodes that already host a replica, even when *Replica Node-Level Soft Anti-Affinity* was disabled. For more details, see [Issue #12251](https://github.com/longhorn/longhorn/issues/12251). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.10.2.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.10.2/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.9.x to v1.10.2.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.10.2/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Feature - [BACKPORT][v1.10.2][FEATURE] Inherit namespace for longhorn-share-manager in FastFailover mode [12245](https://github.com/longhorn/longhorn/issues/12245) - @yangchiu - [BACKPORT][v1.10.2][FEATURE] [Dependency] aws-sdk-go v1.55.7 is EOL as of 2025-07-31 — plan to migrate to v2? [12181](https://github.com/longhorn/longhorn/issues/12181) - @mantissahz @roger-ryao ### Improvement - [BACKPORT][v1.10.2][IMPROVEMENT] Fix V2 Volume CSI Clone Slowness Caused by VolumeAttachment Webhook Blocking [12329](https://github.com/longhorn/longhorn/issues/12329) - @PhanLe1010 @roger-ryao ### Bug - [BACKPORT][v1.10.2][BUG] `instance-manager` on nodes that don't have hard or solid state disk DDOSing cluster DNS server with TXT query `_grpc_config.localhost` [12536](https://github.com/longhorn/longhorn/issues/12536) - @COLDTURNIP @chriscchien - [BACKPORT] Replica rebuild, clone and restore fail, traffic being sent to HTTP proxy [12518](https://github.com/longhorn/longhorn/issues/12518) - @yangchiu @derekbit - [BACKPORT][v1.10.2][BUG] Healthy replica could be deleted unexpectedly after reducing volume's number of replicas [12512](https://github.com/longhorn/longhorn/issues/12512) - @yangchiu @shuo-wu - [BACKPORT][v1.10.2][BUG] Data locality enabled volume fails to remove an existing running replica after numberOfReplicas reduced [12509](https://github.com/longhorn/longhorn/issues/12509) - @derekbit @chriscchien - [BACKPORT][v1.10.2][BUG] System backup may fail to be created or deleted [12479](https://github.com/longhorn/longhorn/issues/12479) - @yangchiu @mantissahz - [BACKPORT][v1.10.2][BUG] Some default settings in questions.yaml are placed incorrectly. [12222](https://github.com/longhorn/longhorn/issues/12222) - @derekbit @roger-ryao - [BACKPORT][v1.10.2][BUG] Auto balance feature may lead to volumes falling into a replica deletion-recreation loop [12482](https://github.com/longhorn/longhorn/issues/12482) - @shuo-wu @roger-ryao - [BACKPORT][v1.10.2][BUG] Single replica volume could get stuck in attaching/detaching loop after the replica node rebooted [12494](https://github.com/longhorn/longhorn/issues/12494) - @COLDTURNIP @yangchiu - [BACKPORT][v1.10.2][BUG] Potential Instance Manager Client Context Leak [12200](https://github.com/longhorn/longhorn/issues/12200) - @derekbit @chriscchien - [BACKPORT][v1.10.2][BUG] SnapshotBack proxy request might be sent to incorrect instance-manager pod [12476](https://github.com/longhorn/longhorn/issues/12476) - @derekbit @chriscchien - [BACKPORT][v1.10.2][BUG] unknown OS condition in node CR is not properly removed during upgrade [12451](https://github.com/longhorn/longhorn/issues/12451) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.10.2][BUG] RWX volume becomes unavailable after drain node [12231](https://github.com/longhorn/longhorn/issues/12231) - @yangchiu @mantissahz - [BACKPORT][v1.10.2][BUG] mounting error is not properly hanedled during CSI node publish volume [12382](https://github.com/longhorn/longhorn/issues/12382) - @COLDTURNIP @yangchiu - [BACKPORT][v1.10.2][BUG] Encrypted Volume Cannot Be Expanded Online [12368](https://github.com/longhorn/longhorn/issues/12368) - @yangchiu @mantissahz - [BACKPORT][v1.10.2][BUG] The auo generated backing image pod name is complained by kubelet [12357](https://github.com/longhorn/longhorn/issues/12357) - @COLDTURNIP @yangchiu - [BACKPORT][v1.10.2][BUG] `tests.test_cloning.test_cloning_basic` fails at msater-head [12342](https://github.com/longhorn/longhorn/issues/12342) - @c3y1huang - [BACKPORT][v1.10.2][Bug] A cloned volume cannot be attached to a workload [12208](https://github.com/longhorn/longhorn/issues/12208) - @yangchiu @PhanLe1010 - [BACKPORT][v1.10.2][BUG] Block Mode Volume Migration Stuck [12312](https://github.com/longhorn/longhorn/issues/12312) - @COLDTURNIP @yangchiu @shuo-wu - [BACKPORT][v1.10.2][BUG] Replica auto balance disk pressure threshold stalled with stopped volumes [12334](https://github.com/longhorn/longhorn/issues/12334) - @c3y1huang @chriscchien - [BACKPORT][v1.10.2][BUG] short name mode is enforcing, but image name longhornio/longhorn-manager:v1.10. │ │ 0 returns ambiguous list [12270](https://github.com/longhorn/longhorn/issues/12270) - @yangchiu - [BACKPORT][v1.10.2][BUG] Replicas accumulate during engine upgrade [12115](https://github.com/longhorn/longhorn/issues/12115) - @c3y1huang @chriscchien - [BACKPORT][v1.10.2][BUG] Potential BackingImageManagerClient Connection and Context Leak [12195](https://github.com/longhorn/longhorn/issues/12195) - @derekbit @chriscchien - [BACKPORT][v1.10.2][BUG] Longhorn ignores `Replica Node Level Soft Anti-Affinity` when auto balance is set to `best-effort` [12251](https://github.com/longhorn/longhorn/issues/12251) - @c3y1huang @chriscchien - [BACKPORT][v1.10.2][BUG] invalid memory address or nil pointer dereference (again) [12234](https://github.com/longhorn/longhorn/issues/12234) - @chriscchien @bachmanity1 - [BACKPORT][v1.10.2][BUG] Request Header Or Cookie Too Large in Web UI with OIDC auth [12213](https://github.com/longhorn/longhorn/issues/12213) - @chriscchien @houhoucoop ## Contributors - @COLDTURNIP - @PhanLe1010 - @bachmanity1 - @c3y1huang - @chriscchien - @derekbit - @houhoucoop - @innobead - @mantissahz - @rebeccazzzz - @roger-ryao - @shuo-wu - @sushant-suse - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.11.0.md ================================================ # Longhorn v1.11.0 Release Notes The Longhorn team is excited to announce the release of Longhorn v1.11.0. This release marks a major milestone, with the **V2 Data Engine** officially entering the **Technical Preview** stage following significant stability improvements. Additionally, this version optimizes the stability of the whole system and introduces critical improvements in resource observability, scheduling, and utilization. For terminology and background on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). > [!WARNING] > > ## Hotfix > > The `longhorn-instance-manager:v1.11.0` image is affected by a [regression issue](https://github.com/longhorn/longhorn/issues/12573) introduced by the new longhorn-instance-manager Proxy service APIs. The bug causes Proxy connection leaks in the longhorn-instance-manager pods, resulting in increased memory usage. To mitigate this issue, replace `longhornio/longhorn-instance-manager:v1.11.0` with the hotfixed image `longhornio/longhorn-instance-manager:v1.11.0-hotfix-1`. > > You can apply the update by following these steps: > > 1. **Update the `longhorn-instance-manager` image** > > - Change the longhorn-instance-manager image tag from `v1.11.0` to `v1.11.0-hotfix-1` in the appropriate file: > - For Helm: Update `values.yaml` > - For manifests: Update the deployment manifest directly. > > 2. **Proceed with the upgrade** > > - Apply the changes using your standard Helm upgrade command or reapply the updated manifest. ## Deprecation ### V2 Backing Image Deprecation The Backing Image feature for the V2 Data Engine is now deprecated in v1.11.0 and is scheduled for removal in v1.12.0. Users using V2 volumes for virtual machines are encouraged to adopt the [Containerized Data Importer (CDI)](https://kubevirt.io/user-guide/operations/containerized_data_importer/) for volume population instead. [GitHub Issue #12237](https://github.com/longhorn/longhorn/issues/12237) ## Primary Highlights ### V2 Data Engine #### Now in Technical Preview Stage We are pleased to announce that the V2 Data Engine has officially graduated to the **Technical Preview** stage. This indicates increased stability and feature maturity as we move toward General Availability. > **Limitation:** While the engine is in Technical Preview, live upgrade is not supported yet. V2 volumes must be detached (offline) before engine upgrade. #### Support for `ublk` Frontend Users can now configure `ublk` (Userspace Block Device) as the frontend for V2 Data Engine volumes. This provides a high-performance alternative to the NVMe-oF frontend for environments running Kernel v6.0+. [GitHub Issue #11039](https://github.com/longhorn/longhorn/issues/11039) ### V1 Data Engine #### Faster Replica Rebuilding from Multiple Sources The V1 Data Engine now supports parallel rebuilding. When a replica needs to be rebuilt, the engine can now stream data from multiple healthy replicas simultaneously rather than a single source. This significantly reduces the time required to restore redundancy for volumes containing tons of scattered data chunks. [GitHub Issue #11331](https://www.google.com/search?q=https://github.com/longhorn/longhorn/issues/11331) ### General #### Balance-Aware Algorithm Disk Selection For Replica Scheduling Longhorn improves the disk selection for the replica scheduling by introducing an intelligent `balance-aware` scheduling algorithm, reducing uneven storage usage across nodes and disks. [GitHub Issue #10512](https://github.com/longhorn/longhorn/issues/10512) #### Node Disk Health Monitoring Longhorn now actively monitors the physical health of the underlying disks used for storage by using S.M.A.R.T. data. This allows administrators to identify issues and raise alerts when abnormal SMART metrics are detected, helping prevent failed volumes. [GitHub Issue #12016](https://github.com/longhorn/longhorn/issues/12016) #### Share Manager Networking Users can now configure an extra network interface for the Share Manager to support complex network segmentation requirements. [GitHub Issue #10269](https://github.com/longhorn/longhorn/issues/10269) #### ReadWriteOncePod (RWOP) Support Full support for the Kubernetes `ReadWriteOncePod` access mode has been added. [GitHub Issue #9727](https://github.com/longhorn/longhorn/issues/9727) #### StorageClass `allowedTopologies` Support Administrators can now use the `allowedTopologies` field in Longhorn StorageClasses to restrict volume provisioning to specific zones, regions, or nodes within the cluster. [GitHub Issue #12261](https://github.com/longhorn/longhorn/issues/12261) ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.11.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.11.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.10.x to v1.11.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.11.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues in this release ### Highlight - [FEATURE] Add support for ReadWriteOncePod access mode [9727](https://github.com/longhorn/longhorn/issues/9727) - @derekbit @shikanime @chriscchien @Copilot - [FEATURE] Scale replica rebuilding speed from multiple healthy replicas [11331](https://github.com/longhorn/longhorn/issues/11331) - @derekbit @shuo-wu @roger-ryao @Copilot - [FEATURE] Support StorageClass allowedTopologies for Longhorn volumes [12261](https://github.com/longhorn/longhorn/issues/12261) - @yangchiu @derekbit @hookak @Copilot - [FEATURE] Support extra network interface (not only storage network) on the share manager pod [10269](https://github.com/longhorn/longhorn/issues/10269) - @yangchiu @c3y1huang - [FEATURE] Monitor Node Disk Health [12016](https://github.com/longhorn/longhorn/issues/12016) - @c3y1huang @roger-ryao - [FEATURE] Replica Auto Balance Across Nodes based on Node Disk Space Consumption [10512](https://github.com/longhorn/longhorn/issues/10512) - @davidcheng0922 @chriscchien ### Feature - [FEATURE] Guess Linux distro from the package manager [12153](https://github.com/longhorn/longhorn/issues/12153) - @yangchiu @derekbit @NamrathShetty @Copilot - [FEATURE] Provide a helm chart setting to define the managerUrl [10583](https://github.com/longhorn/longhorn/issues/10583) - @lexfrei @yangchiu - [FEATURE] Add metric for last backup of a volume [6049](https://github.com/longhorn/longhorn/issues/6049) - @c3y1huang @roger-ryao - [FEATURE] Real-time volume performance monitoring [368](https://github.com/longhorn/longhorn/issues/368) - @derekbit @hookak - [UI][FEATURE] Monitor Node Disk Health [12263](https://github.com/longhorn/longhorn/issues/12263) - @houhoucoop @roger-ryao - [FEATURE] custom annotation/label of UI's k8s service on value.yaml of helm chart [11754](https://github.com/longhorn/longhorn/issues/11754) - @yangchiu @lucasl0st - [FEATURE] Make `longhornctl` load `ublk_drv` module when kernel version is 6 or newer [11803](https://github.com/longhorn/longhorn/issues/11803) - @chriscchien @bachmanity1 - [BUG] Inherit namespace for longhorn-share-manager in FastFailover mode [12244](https://github.com/longhorn/longhorn/issues/12244) - @yangchiu @semenas - [FEATURE] Enable CSI pod anti-affinity preset update [12100](https://github.com/longhorn/longhorn/issues/12100) - @yangchiu @yulken - [FEATURE] [Dependency] aws-sdk-go v1.55.7 is EOL as of 2025-07-31 — plan to migrate to v2? [12098](https://github.com/longhorn/longhorn/issues/12098) - @mantissahz @roger-ryao - [FEATURE] Change volume operation menu button behaviour from hover to click. [11408](https://github.com/longhorn/longhorn/issues/11408) - @yangchiu @houhoucoop - [FEATURE] "hard" podAntiAffinity for csi-attacher/csi-provisioner/csi-resizer/csi-snapshotter [11617](https://github.com/longhorn/longhorn/issues/11617) - @yangchiu @yulken - [FEATURE] node storage scheduled metrics [11949](https://github.com/longhorn/longhorn/issues/11949) - @yangchiu @AoRuiAC ### Improvement - [IMPROVEMENT] Generalize the offline rebuilding setting for both data engines [12484](https://github.com/longhorn/longhorn/issues/12484) - @mantissahz @chriscchien - [IMPROVEMENT] Introduce Concurrent Job Limit for Snapshot Operations [11635](https://github.com/longhorn/longhorn/issues/11635) - @yangchiu @derekbit @davidcheng0922 @Copilot - [IMPROVEMENT] Improve disk error logging to retain errors from newDiskServiceClients() [12446](https://github.com/longhorn/longhorn/issues/12446) - @yangchiu @davidcheng0922 - [IMPROVEMENT] Propagate longhorn-manager's timezone to instance-manager and CSI pods [12448](https://github.com/longhorn/longhorn/issues/12448) - @hookak @roger-ryao - [UI][FEATURE] Scale replica rebuilding speed from multiple healthy replicas [12461](https://github.com/longhorn/longhorn/issues/12461) - @houhoucoop @roger-ryao - [IMPROVEMENT] Configure rolling update strategy for longhorn-manager and CSI deployments [12240](https://github.com/longhorn/longhorn/issues/12240) - @hookak @chriscchien - [IMPROVEMENT] Improve log messages for `rebuildNewReplica()` in longhorn-manager [12426](https://github.com/longhorn/longhorn/issues/12426) - @derekbit @chriscchien - [IMPROVEMENT] misleading message when instance manager tries to create the pod [11759](https://github.com/longhorn/longhorn/issues/11759) - @mantissahz @chriscchien - [IMPROVEMENT] To improve the debugging process and UX, it would be nice that the error is recorded in the `instancemanager.status.conditions`. [6732](https://github.com/longhorn/longhorn/issues/6732) - @mantissahz @chriscchien - [IMPROVEMENT] Add setting to disable node disk health monitoring [12300](https://github.com/longhorn/longhorn/issues/12300) - @derekbit @roger-ryao @Copilot - [IMPROVEMENT] Avoid repeat engine restart when there are replica unavailable during migration [11397](https://github.com/longhorn/longhorn/issues/11397) - @yangchiu @shuo-wu - [IMPROVEMENT] [Script] Minor script adjustments from PR #12177 [12187](https://github.com/longhorn/longhorn/issues/12187) - @rauldsl @yangchiu - [IMPROVEMENT] Check toolchain versions before generate k8s codes [12164](https://github.com/longhorn/longhorn/issues/12164) - @derekbit @roger-ryao - [IMPROVEMENT] Create Volume UI improvement, Automatically Filter `Data Source` Based on v1 or v2 Selection [11846](https://github.com/longhorn/longhorn/issues/11846) - @yangchiu @houhoucoop - [IMPROVEMENT] Disable the snapshot of v1 volume hashing while it is being deleted [10294](https://github.com/longhorn/longhorn/issues/10294) - @davidcheng0922 @chriscchien - [IMPROVEMENT] Expose SPDK UBLK Parameters [11039](https://github.com/longhorn/longhorn/issues/11039) - @derekbit @PhanLe1010 @roger-ryao @Copilot - [IMPROVEMENT] Check that block device is not in use before creating disk [12078](https://github.com/longhorn/longhorn/issues/12078) - @chriscchien @bachmanity1 - [UI][IMPROVEMENT] Awareness of when an offline replica rebuilding is triggered for an individual volume [11247](https://github.com/longhorn/longhorn/issues/11247) - @houhoucoop @roger-ryao - [IMPROVEMENT] Ensure synchronized upgrades between longhorn-manager and instance-manager [12309](https://github.com/longhorn/longhorn/issues/12309) - @hookak @chriscchien - [IMPROVEMENT] Add Resource Limits Configuration for Longhorn manager/instance-manager [12225](https://github.com/longhorn/longhorn/issues/12225) - @hookak @chriscchien - [IMPROVEMENT] Add Validation Webhook to Volume Expansion When Node Disk Is Full [12134](https://github.com/longhorn/longhorn/issues/12134) - @yangchiu @davidcheng0922 - [UI][IMPROVEMENT] Expose SPDK UBLK Parameters [12166](https://github.com/longhorn/longhorn/issues/12166) - @houhoucoop @roger-ryao - [IMPROVEMENT] Fix V2 Volume CSI Clone Slowness Caused by VolumeAttachment Webhook Blocking [12328](https://github.com/longhorn/longhorn/issues/12328) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Use label-based state in metrics instead of numeric values [10723](https://github.com/longhorn/longhorn/issues/10723) - @hookak @roger-ryao - [IMPROVEMENT] Add Resource Limits Configuration for CSI Components [12224](https://github.com/longhorn/longhorn/issues/12224) - @yangchiu @hookak @Copilot - [IMPROVEMENT] Awareness of when an offline replica rebuilding is triggered for an individual volume [11246](https://github.com/longhorn/longhorn/issues/11246) - @yangchiu @mantissahz - [IMPROVEMENT] Add loadBalancerClass value inside a helm chart for ui service [12273](https://github.com/longhorn/longhorn/issues/12273) - @ehpc @chriscchien - [IMPROVEMENT] Add DNS round-robin load balancing to the pool of S3 addresses [12296](https://github.com/longhorn/longhorn/issues/12296) - @yangchiu - [UI][IMPROVEMENT] Should Not Hide the Deleted Snapshots on UI [11620](https://github.com/longhorn/longhorn/issues/11620) - @yangchiu @houhoucoop - [IMPROVEMENT] Helm chart Multiple TLS FQDNs [12127](https://github.com/longhorn/longhorn/issues/12127) - @yangchiu @hrabalvojta - [IMPROVEMENT] Removing executables from mirrored-longhornio-longhorn-engine image [11254](https://github.com/longhorn/longhorn/issues/11254) - @derekbit @chriscchien - [IMPROVEMENT] [DOC] Clarify replica auto-balance behavior for unhealthy and detached volumes [12002](https://github.com/longhorn/longhorn/issues/12002) - @roger-ryao @sushant-suse - [IMPROVEMENT] CRD enum values [9718](https://github.com/longhorn/longhorn/issues/9718) - @roger-ryao @nzhan126 - [DOC] Troubleshooting KB Articles Fix Typos [12199](https://github.com/longhorn/longhorn/issues/12199) - @jmeza-xyz - [IMPROVEMENT] Remove backupstore related settings [11026](https://github.com/longhorn/longhorn/issues/11026) - @nzhan126 - [IMPROVEMENT] Reject Trim Operation on Block Volume [12048](https://github.com/longhorn/longhorn/issues/12048) - @yangchiu @derekbit - [IMPROVEMENT] Replace `github.com/pkg/errors` with `github.com/cockroachdb/errors` [11413](https://github.com/longhorn/longhorn/issues/11413) - @derekbit @chriscchien - [UI][IMPROVEMENT] UI shows the backing image virtual size [11674](https://github.com/longhorn/longhorn/issues/11674) - @chriscchien @houhoucoop - [IMPROVEMENT] Simplify locking in unsub and stream methods [12057](https://github.com/longhorn/longhorn/issues/12057) - @derekbit @NamrathShetty - [UI][IMPROVEMENT] Show Error Message for Unschedulable Disks [11449](https://github.com/longhorn/longhorn/issues/11449) - @yangchiu @houhoucoop - [IMPROVEMENT] The `auto-delete-pod-when-volume-detached-unexpectedly` should only focus on the Kubernetes builtin workload. [12120](https://github.com/longhorn/longhorn/issues/12120) - @derekbit @chriscchien @sushant-suse - [IMPROVEMENT] `CSIStorageCapacity` objects must show schedulable (allocatable) capacity [12014](https://github.com/longhorn/longhorn/issues/12014) - @chriscchien @bachmanity1 - [IMPROVEMENT] improve error logging for failed mounting during node publish volume [12025](https://github.com/longhorn/longhorn/issues/12025) - @COLDTURNIP @roger-ryao - [IMPROVEMENT] Improve Helm Chart defaultSettings handling with automatic quoting and multi-type support [12019](https://github.com/longhorn/longhorn/issues/12019) - @derekbit @chriscchien - [IMPROVEMENT] volume `spec.backingImage` and `spec.encrypted` shouldn't allow to update for both v1 and v2 data engines [11615](https://github.com/longhorn/longhorn/issues/11615) - @yulken @roger-ryao ### Bug - [BUG] V2 DR volume failed if backupstore is temporarily unavailable after node reboot [12543](https://github.com/longhorn/longhorn/issues/12543) - @c3y1huang @roger-ryao - [BUG] SnapshotBack proxy request might be sent to incorrect instance-manager pod [12475](https://github.com/longhorn/longhorn/issues/12475) - @derekbit @chriscchien - [BUG] Replica rebuild, clone and restore fail, traffic being sent to HTTP proxy [12304](https://github.com/longhorn/longhorn/issues/12304) - @derekbit @chriscchien @roger-ryao - [BUG] `instance-manager` on nodes that don't have hard or solid state disk DDOSing cluster DNS server with TXT query `_grpc_config.localhost` [12521](https://github.com/longhorn/longhorn/issues/12521) - @COLDTURNIP @chriscchien - [BUG][v1.11.0-rc3] test_basic.py::test_snapshot fails on v2 data engine [12526](https://github.com/longhorn/longhorn/issues/12526) - @derekbit @chriscchien - [BUG] RWX volume causes process uninterruptible sleep [11907](https://github.com/longhorn/longhorn/issues/11907) - @COLDTURNIP @chriscchien - [BUG] Healthy replica could be deleted unexpectedly after reducing volume's number of replicas [12511](https://github.com/longhorn/longhorn/issues/12511) - @yangchiu @shuo-wu - [BUG] Auto balance feature may lead to volumes falling into a replica deletion-recreation loop [11730](https://github.com/longhorn/longhorn/issues/11730) - @shuo-wu @roger-ryao - [BUG] Data locality enabled volume fails to remove an existing running replica after numberOfReplicas reduced [12488](https://github.com/longhorn/longhorn/issues/12488) - @derekbit @chriscchien - [BUG] Single replica volume could get stuck in attaching/detaching loop after the replica node rebooted [9141](https://github.com/longhorn/longhorn/issues/9141) - @COLDTURNIP @yangchiu - [BUG] v2 volume rebuild performance doesn't improve after enabling snapshot integrity [12416](https://github.com/longhorn/longhorn/issues/12416) - @yangchiu @davidcheng0922 - [BUG] Request Header Or Cookie Too Large in Web UI with OIDC auth [12077](https://github.com/longhorn/longhorn/issues/12077) - @chriscchien @houhoucoop - [BUG] v1.11.x upgrade test may fail because the default disk of a node is removed during a test case and cannot be re-added [12469](https://github.com/longhorn/longhorn/issues/12469) - @COLDTURNIP @yangchiu - [BUG] Potential Instance Manager Client Context Leak [12198](https://github.com/longhorn/longhorn/issues/12198) - @derekbit @chriscchien - [BUG] v2 DR volume becomes faulted during incremental restoration after source volume expansion [12465](https://github.com/longhorn/longhorn/issues/12465) - @yangchiu @davidcheng0922 - [BUG] `rebuildConcurrentSyncLimit` field is omitted from `volume.spec` when value is `0` [12471](https://github.com/longhorn/longhorn/issues/12471) - @derekbit @houhoucoop @roger-ryao - [BUG] Adding multiple disks to the same node concurrently may occasionally fail [11971](https://github.com/longhorn/longhorn/issues/11971) - @davidcheng0922 @roger-ryao - [BUG] unknown OS condition in node CR is not properly removed during upgrade [12450](https://github.com/longhorn/longhorn/issues/12450) - @COLDTURNIP @roger-ryao - [BUG] Longhorn charts does not take care timezone [11965](https://github.com/longhorn/longhorn/issues/11965) - @hookak @roger-ryao - [BUG] Pod failed to use an activated DR volume, got `UNEXPECTED INCONSISTENCY; RUN fsck MANUALLY` error [12444](https://github.com/longhorn/longhorn/issues/12444) - @yangchiu - [BUG] v2 volumes do not reuse failed replicas for rebuilding as expected [12413](https://github.com/longhorn/longhorn/issues/12413) - @yangchiu @shuo-wu - [BUG] v2 volumes complete offline rebuilding with an extra failed replica if a node is rebooted during the rebuild [12407](https://github.com/longhorn/longhorn/issues/12407) - @yangchiu @mantissahz - [BUG] Test case `test_rebuild_failure_with_intensive_data` is failing because replicas cannot be rebuilt after replica process crashed [12436](https://github.com/longhorn/longhorn/issues/12436) - @yangchiu @shuo-wu - [BUG] Replica mode becomes empty and replica rebuilding cannot be triggered after upgrading from v1.10.1 to master-head or v1.11.0-rc1 [12431](https://github.com/longhorn/longhorn/issues/12431) - @yangchiu @derekbit - [BUG] v2 volumes get stuck in `Attaching/Detaching` loop after node reboots [12406](https://github.com/longhorn/longhorn/issues/12406) - @yangchiu @c3y1huang - [BUG] test_basic.py::test_expansion_basic is flaky on v2 data engine due to revert snapshot fail [12235](https://github.com/longhorn/longhorn/issues/12235) - @davidcheng0922 @chriscchien - [BUG] Longhorn nodes may fail to recover after node reboots [12422](https://github.com/longhorn/longhorn/issues/12422) - @COLDTURNIP @yangchiu - [BUG] Missing `Frontend` default value when creating v2 volumes via Longhorn UI [12152](https://github.com/longhorn/longhorn/issues/12152) - @houhoucoop @roger-ryao - [BUG] setting values are not converted to strings in Longhorn UI [12192](https://github.com/longhorn/longhorn/issues/12192) - @chriscchien @houhoucoop - [BUG] `disk health information` appears briefly [12415](https://github.com/longhorn/longhorn/issues/12415) - @c3y1huang @roger-ryao - [BUG] encrypted v2 volume gets stuck in `Attaching/Detaching` loop after volume expansion [12359](https://github.com/longhorn/longhorn/issues/12359) - @yangchiu @davidcheng0922 - [BUG] Unexpected orphaned replica is created after node reboot, preventing new replica from being scheduled on that node, and blocking v2 volume from recovering to healthy state [11333](https://github.com/longhorn/longhorn/issues/11333) - @yangchiu @c3y1huang - [BUG] RWX volume becomes unavailable after drain node [12226](https://github.com/longhorn/longhorn/issues/12226) - @yangchiu @mantissahz - [BUG] invalid memory address or nil pointer dereference [11939](https://github.com/longhorn/longhorn/issues/11939) - @bachmanity1 @roger-ryao - [BUG] share-manager excessive memory usage [11938](https://github.com/longhorn/longhorn/issues/11938) - @derekbit @chriscchien - [BUG] Encrypted Volume Cannot Be Expanded Online [12366](https://github.com/longhorn/longhorn/issues/12366) - @yangchiu @chriscchien - [BUG] Backing image download gets stuck after network disconnection [11622](https://github.com/longhorn/longhorn/issues/11622) - @COLDTURNIP @chriscchien - [BUG] Can not delete the parent of volume head snapshot of a v2 volume [9064](https://github.com/longhorn/longhorn/issues/9064) - @yulken @chriscchien - [BUG] changing of the volume controller owner caused: BUG: multiple engines detected when volume is detached [1755](https://github.com/longhorn/longhorn/issues/1755) - @PhanLe1010 @chriscchien - [BUG] mounting error is not properly handled during CSI node publish volume [12006](https://github.com/longhorn/longhorn/issues/12006) - @COLDTURNIP @yangchiu - [BUG] test_rebuild_after_replica_file_crash failed on master-head [12389](https://github.com/longhorn/longhorn/issues/12389) - @derekbit @chriscchien - [BUG] `test_backing_image_auto_resync` is flaky due to recent commit [12387](https://github.com/longhorn/longhorn/issues/12387) - @derekbit @chriscchien - [BUG] Flooding messages `Failed to resolve sysfs path for \"/sys/class/block/root\ ...` in longhorn-manager [12344](https://github.com/longhorn/longhorn/issues/12344) - @c3y1huang @roger-ryao - [BUG] v2 volumes could fail to auto salvage after cluster restart [11336](https://github.com/longhorn/longhorn/issues/11336) - @yangchiu @c3y1huang - [BUG] The auo generated backing image pod name is complained by kubelet [12356](https://github.com/longhorn/longhorn/issues/12356) - @COLDTURNIP @yangchiu - [BUG] `test_restore_inc_with_offline_expansion` fails on v2 data engine [12313](https://github.com/longhorn/longhorn/issues/12313) - @davidcheng0922 @chriscchien - [BUG] Block disks have a chance become Unschedulable in v2 regression test in test_rebuild_with_restoration [11446](https://github.com/longhorn/longhorn/issues/11446) - @shuo-wu @chriscchien - [BUG] v2 volume workload FailedMount with message Staging target path `/var/lib/kubelet/plugins/kubernetes.io/csi/driver.longhorn.io/xxx/globalmount is no longer valid` [10476](https://github.com/longhorn/longhorn/issues/10476) - @yangchiu @shuo-wu - [BUG] [v1.10.0-rc1] v2 DR volume stuck Unhealthy after incremental restore with replica deletion(`test_rebuild_with_inc_restoration`) [11684](https://github.com/longhorn/longhorn/issues/11684) - @c3y1huang @chriscchien - [BUG] `test_data_locality_strict_local_node_affinity` fails at master-head [12343](https://github.com/longhorn/longhorn/issues/12343) - @derekbit @chriscchien - [BUG] `tests.test_cloning.test_cloning_basic` fails at msater-head [12341](https://github.com/longhorn/longhorn/issues/12341) - @derekbit @chriscchien @Copilot - [BUG] v2 volume could get stuck in `Detaching` indefinitely after node reboot [11332](https://github.com/longhorn/longhorn/issues/11332) - @yangchiu @c3y1huang - [Bug] A cloned volume cannot be attached to a workload [12206](https://github.com/longhorn/longhorn/issues/12206) - @yangchiu @PhanLe1010 - [BUG] Block Mode Volume Migration Stuck [12311](https://github.com/longhorn/longhorn/issues/12311) - @COLDTURNIP @yangchiu @shuo-wu - [BUG] Replica auto balance disk pressure threshold stalled with stopped volumes [10837](https://github.com/longhorn/longhorn/issues/10837) - @c3y1huang @chriscchien - [BUG] short name mode is enforcing, but image name longhornio/longhorn-manager:v1.10. │ │ 0 returns ambiguous list [12268](https://github.com/longhorn/longhorn/issues/12268) - @yangchiu @Wqrld - [BUG] invalid memory address or nil pointer dereference (again) [12233](https://github.com/longhorn/longhorn/issues/12233) - @chriscchien @bachmanity1 - [BUG] Restored v2 volume gets stuck in `RestoreInProgress` state if backup is deleted during restoration [11828](https://github.com/longhorn/longhorn/issues/11828) - @yangchiu @c3y1huang - [BUG] spdk_tgt is crashed due to SIGSEGV [11698](https://github.com/longhorn/longhorn/issues/11698) - @c3y1huang - [BUG] Longhorn ignores `Replica Node Level Soft Anti-Affinity` when auto balance is set to `best-effort` [11189](https://github.com/longhorn/longhorn/issues/11189) - @c3y1huang @chriscchien - [BUG] SPDK NVMe synchronous calls [11096](https://github.com/longhorn/longhorn/issues/11096) - - [BUG] Replicas accumulate during engine upgrade [12111](https://github.com/longhorn/longhorn/issues/12111) - @c3y1huang @chriscchien - [BUG] Some default settings in questions.yaml are placed incorrectly. [12219](https://github.com/longhorn/longhorn/issues/12219) - @derekbit @roger-ryao - [BUG] Chart does not handle defaultSettings.taintToleration with a trailing colon [12162](https://github.com/longhorn/longhorn/issues/12162) - @derekbit @chriscchien - [BUG] Fix SPDK v25.05 CVE issue [11969](https://github.com/longhorn/longhorn/issues/11969) - @derekbit @roger-ryao - [BUG] Potential BackingImageManagerClient Connection and Context Leak [12194](https://github.com/longhorn/longhorn/issues/12194) - @derekbit @chriscchien - [BUG] Instance manager pod `awsIAMRoleArn` annotation disappearing [9923](https://github.com/longhorn/longhorn/issues/9923) - @yangchiu @mantissahz - [BUG] Node block-type disk is unable to unbind after Longhorn uninstall [9127](https://github.com/longhorn/longhorn/issues/9127) - @yangchiu @davidcheng0922 - [BUG] longhorn-manager fails to start after upgrading from 1.9.2 to 1.10.0 [11864](https://github.com/longhorn/longhorn/issues/11864) - @derekbit @roger-ryao - [BUG][UI] When creating volume/backing image, change `Data Engine` will reset `Number of Replicas` [11775](https://github.com/longhorn/longhorn/issues/11775) - @yangchiu @houhoucoop - [BUG] Backup target metric is broken [12073](https://github.com/longhorn/longhorn/issues/12073) - @mantissahz @roger-ryao - [BUG] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 at longhorn-engine/pkg/controller/control.go:218 +0x2de [12081](https://github.com/longhorn/longhorn/issues/12081) - @liyimeng @roger-ryao - [BUG] Unable to complete uninstallation due to the remaining backuptarget [11934](https://github.com/longhorn/longhorn/issues/11934) - @mantissahz @roger-ryao - [BUG] NVME disk not found in v2 data engine (failed to find device for BDF) [11903](https://github.com/longhorn/longhorn/issues/11903) - @derekbit @roger-ryao - [BUG] NPE error during recurring job execution [11925](https://github.com/longhorn/longhorn/issues/11925) - @yangchiu @shuo-wu - [BUG] v2 volume creation failed on talos nodes [11910](https://github.com/longhorn/longhorn/issues/11910) - @c3y1huang @chriscchien - [BUG] DR volume gets stuck in `unknown` state if engine image is deleted from the attached node [11995](https://github.com/longhorn/longhorn/issues/11995) - @yangchiu @shuo-wu - [BUG] Volume gets stuck in `attaching` state if engine image image is not deployed on one of nodes [11994](https://github.com/longhorn/longhorn/issues/11994) - @yangchiu @shuo-wu - [BUG] Rebooting the volume attached node during a v2 DR volume incremental restoration, the restoration is left incomplete and the activation has no effect [11778](https://github.com/longhorn/longhorn/issues/11778) - @yangchiu @c3y1huang - [BUG] Unable to detach a v2 volume after labeling `disable-v2-data-engine=true` [11799](https://github.com/longhorn/longhorn/issues/11799) - @yangchiu @mantissahz - [BUG] `test_system_backup_and_restore` test case failed on master-head [11933](https://github.com/longhorn/longhorn/issues/11933) - @derekbit @chriscchien - [BUG] Shebang refactor in scripts may cause compatibility issues [11815](https://github.com/longhorn/longhorn/issues/11815) - @NamrathShetty @chriscchien - [BUG] longhorn-spdk-engine CIs complain that the unit tests successfully hash system created snapshots [11822](https://github.com/longhorn/longhorn/issues/11822) - @yangchiu @shuo-wu - [BUG] Unable to re-add block-type disks by BDF after re-enable v2 data engine [11860](https://github.com/longhorn/longhorn/issues/11860) - @yangchiu @davidcheng0922 - [BUG] V2 volume stuck in volume attachment (V2 interrupt mode) [11816](https://github.com/longhorn/longhorn/issues/11816) - @c3y1huang - [BUG] Goroutine leak in instance-manager when using v2 data engine [11959](https://github.com/longhorn/longhorn/issues/11959) - @PhanLe1010 @chriscchien - [BUG] csi-provisioner silently fails to create CSIStorageCapacity if dataEngine parameter is missing [11906](https://github.com/longhorn/longhorn/issues/11906) - @yangchiu @bachmanity1 - [BUG][v1.8.x] v2 volume stuck at attaching due to stopped replica [10486](https://github.com/longhorn/longhorn/issues/10486) - @chriscchien - [BUG] longhorn-engine's UI panics [11867](https://github.com/longhorn/longhorn/issues/11867) - @derekbit @chriscchien @Copilot - [BUG] v2 volume workload gets stuck in `ContainerCreating` or `Unknown` state with `FailedMount` error [10111](https://github.com/longhorn/longhorn/issues/10111) - @yangchiu @shuo-wu - [BUG] Volume is unable to upgrade if the number of active replicas is larger than `volume.spec.numberOfReplicas` [11825](https://github.com/longhorn/longhorn/issues/11825) - @yangchiu @derekbit - [BUG] UI fails to deploy when only IPv4 is enabled on nodes with v1.10.0 version [11869](https://github.com/longhorn/longhorn/issues/11869) - @yangchiu @c3y1huang - [BUG] v2 DR volume fails to auto-reattach when engine image missing on current node [11772](https://github.com/longhorn/longhorn/issues/11772) - @chriscchien - [BUG] inconsistent behavior of v2 volume after labeling disable-v2-data-engine to the volume attached node and deleting the instance manager [11578](https://github.com/longhorn/longhorn/issues/11578) - @yangchiu ### Misc - [DOC] Fix Talos install documentation for current versions [12514](https://github.com/longhorn/longhorn/issues/12514) - - [DOC] Add KB article for the failure of RWX volume detachment [12238](https://github.com/longhorn/longhorn/issues/12238) - @sushant-suse - [TASK] Fix flaky regression test case `test_recurring_job.py::test_recurring_job_snapshot_cleanup` for v2 data engine [12464](https://github.com/longhorn/longhorn/issues/12464) - @derekbit @chriscchien - [DOC] Review and Update Ingress Controller Examples for Longhorn UI [12252](https://github.com/longhorn/longhorn/issues/12252) - @yangchiu @sushant-suse - [DOC] Incorrect longhornctl subcommand [12423](https://github.com/longhorn/longhorn/issues/12423) - @chriscchien @roger-ryao - [TASK] Update nvme and libnvme to v2.16 and v1.16.1 [12391](https://github.com/longhorn/longhorn/issues/12391) - @derekbit @chriscchien - [DOC] Disk Aggregation Options [12378](https://github.com/longhorn/longhorn/issues/12378) - @davidcheng0922 @roger-ryao - [TASK] Deprecate V2 Backing Image Feature [12237](https://github.com/longhorn/longhorn/issues/12237) - @derekbit @chriscchien - feat(chart): Add Gateway API HTTPRoute support for Longhorn UI [12299](https://github.com/longhorn/longhorn/issues/12299) - @lexfrei @derekbit @chriscchien @Copilot - [DOC] V2 data engine: delete snapshot after volume-head behaves inconsistently vs v1 [12355](https://github.com/longhorn/longhorn/issues/12355) - @chriscchien @sushant-suse - [TASK] Revert Base Image bci-base:16.0 to bci-base:15.7 [12354](https://github.com/longhorn/longhorn/issues/12354) - @derekbit @chriscchien - [DOC] Clarify share-manager image update behavior after system upgrade with attached RWX volumes [12363](https://github.com/longhorn/longhorn/issues/12363) - @derekbit @chriscchien - [DOC] Clarify expected behavior of old instance manager pods after live engine upgrade [12361](https://github.com/longhorn/longhorn/issues/12361) - @derekbit @chriscchien - [TASK] Update Longhorn v1.11.0 SPDK to v25.09 [11975](https://github.com/longhorn/longhorn/issues/11975) - @derekbit @chriscchien - [TASK] Bump Longhorn Component `registry.suse.com/bci/bci-base` to 16.0 [12145](https://github.com/longhorn/longhorn/issues/12145) - @derekbit @chriscchien - [DOC] Add KB Article: Handling Persistent Replica Failures via Disk Isolation [12242](https://github.com/longhorn/longhorn/issues/12242) - @derekbit @roger-ryao - [DOC] Document how to permanently enable hugepages [12167](https://github.com/longhorn/longhorn/issues/12167) - @roger-ryao @sushant-suse - [DOC] Update existing terminologies and add new terminologies [12302](https://github.com/longhorn/longhorn/issues/12302) - @sushant-suse - [DOC] Add a KB for restoring data from an orphan replica directory [9972](https://github.com/longhorn/longhorn/issues/9972) - @yangchiu @sushant-suse - [DOC] [UI][IMPROVEMENT] Should Not Hide the Deleted Snapshots on UI #11620 [12214](https://github.com/longhorn/longhorn/issues/12214) - @chriscchien @sushant-suse - [DOC] Add `Enterprise` Page in Longhorn Official Document [12110](https://github.com/longhorn/longhorn/issues/12110) - @sushant-suse - [DOC] Update Talos Linux Support with Longhorn [12108](https://github.com/longhorn/longhorn/issues/12108) - @roger-ryao @egrosdou01 - [DOC] [FEATURE] Add support for ReadWriteOncePod access mode [12228](https://github.com/longhorn/longhorn/issues/12228) - @chriscchien - [DOC] Workaround KB doc for backing image manager disk UUID collision issue [12114](https://github.com/longhorn/longhorn/issues/12114) - @COLDTURNIP @roger-ryao - [TASK] Remove testing credentials from backup target manifest examples [11076](https://github.com/longhorn/longhorn/issues/11076) - @davidcheng0922 @roger-ryao - [DOC] Document the Migratable RWX Volume in the Official Document [11277](https://github.com/longhorn/longhorn/issues/11277) - @derekbit @chriscchien @sushant-suse - [DOC][UI][IMPROVEMENT] Show Error Message for Unschedulable Disks #11449 [12151](https://github.com/longhorn/longhorn/issues/12151) - @yangchiu @sushant-suse - [TASK] Create a GitHub Action to Update Versions in longhorn/dev-versions [12062](https://github.com/longhorn/longhorn/issues/12062) - @derekbit - [DOC] Update NFSv4 client installation docs to verify actual NFS version in use [11944](https://github.com/longhorn/longhorn/issues/11944) - @derekbit @chriscchien - [REFACTOR] SAST checks for UI component [11540](https://github.com/longhorn/longhorn/issues/11540) - @sminux @chriscchien - [DOC] Update Longhorn README file [10891](https://github.com/longhorn/longhorn/issues/10891) - @divya-mohan0209 - [BUG] Block disk deletion fails without error message [11952](https://github.com/longhorn/longhorn/issues/11952) - @davidcheng0922 @roger-ryao - [REFACTOR] Remove redundant assignment [11705](https://github.com/longhorn/longhorn/issues/11705) - @jvanz - [TASK] Remove deprecated instances field and instance type from instance manager CR [5844](https://github.com/longhorn/longhorn/issues/5844) - @derekbit @chriscchien - [DOC] Update deployment links according to the document version [11847](https://github.com/longhorn/longhorn/issues/11847) - @yulken ## New Contributors * @ADN182 * @AoRuiAC * @Henllage-hqb * @Mmx233 * @NamrathShetty * @Wqrld * @adegoodyer * @ah8ad3 * @boomam * @brandboat * @bvankampen * @danielskowronski * @davepgreene * @egrosdou01 * @ehpc * @enterdv * @fatihmete * @hrabalvojta * @inqode-lars * @jmeza-xyz * @jvanz * @kocmoc1 * @koeberlue * @lexfrei * @lucasl0st * @madeITBelgium * @marnixbouhuis * @mattn * @maximemoreillon * @mo124121 * @nachtschatt3n * @rajeshkio * @rauldsl * @saimikiry * @sdre15 * @semenas * @shikanime * @sminux * @zijiren233 ## Contributors Thank you to the following contributors who made this release possible. > **Note:** Starting from v1.11.0, as long as a GitHub issue is resolved in the current release, the corresponding authors will be listed in this contributor list as well. If there is still a missing, please contact Longhorn team for the update. - @ADN182 - @AoRuiAC - @COLDTURNIP - @DamiaSan - @Henllage-hqb - @Mmx233 - @NRCan-LGariepy - @NamrathShetty - @PhanLe1010 - @Vicente-Cheng - @WebberHuang1118 - @Wqrld - @adegoodyer - @ah8ad3 - @bachmanity1 - @boomam - @brandboat - @bvankampen - @c3y1huang - @chriscchien - @danielskowronski - @davepgreene - @davidcheng0922 - @derekbit - @dhedberg - @divya-mohan0209 - @egrosdou01 - @ehpc - @enterdv - @fatihmete - @fmunteanu - @forbesguthrie - @hoo29 - @hookak - @houhoucoop - @hrabalvojta - @innobead - @inqode-lars - @james-munson - @jmeza-xyz - @jvanz - @kocmoc1 - @koeberlue - @lexfrei - @liyimeng - @lucasl0st - @madeITBelgium - @mantissahz - @marnixbouhuis - @mattn - @maximemoreillon - @mcerveny - @mo124121 - @nachtschatt3n - @nzhan126 - @rajeshkio - @rauldsl - @rebeccazzzz - @roger-ryao - @runningman84 - @saimikiry - @sdre15 - @semenas - @shikanime - @shuo-wu - @sminux - @sushant-suse - @w13915984028 - @yangchiu - @yasker - @yulken - @zijiren233 ================================================ FILE: CHANGELOG/CHANGELOG-1.11.1.md ================================================ # Longhorn v1.11.1 Release Notes Longhorn v1.11.1 is a patch release that focuses on critical bug fixes, security hardening, and stability improvements for both V1 and V2 data engines. Key highlights include a fix for a significant memory leak in the instance manager and improvements to backup reliability and volume scheduling. We welcome feedback and contributions to help continuously improve Longhorn. For terminology and context on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). ## Important Fixes This release includes several critical stability fixes. ### Longhorn workload pods memory leak Fixed a critical regression where proxy connection leaks in the longhorn-instance-manager pods caused high memory consumption. For more details, see [#12575](https://github.com/longhorn/longhorn/issues/12575) ### Backup & Restore compatibility fix Resolved compatibility issues introduced by aws-go-sdk v2, including backups to S3-compatible storage (like Storj or Google Cloud Storage). This fix ensures the completion of large data transfers to remote backup targets with correct authorization. For more details, see [#12714](https://github.com/longhorn/longhorn/issues/12714) and [12688](https://github.com/longhorn/longhorn/issues/12688) ### V2 Data Engine (SPDK) refinements Several enhancements were delivered for some V2 Data Engine features, including fast replica rebuild and clone. For more details, see [#12751](https://github.com/longhorn/longhorn/issues/12751) and [12748](https://github.com/longhorn/longhorn/issues/12748) ### CSI scheduling enhancement Support CSI topology-aware PV nodeAffinity control. For more details, see [#12689](https://github.com/longhorn/longhorn/issues/12689) and [12656](https://github.com/longhorn/longhorn/issues/12656) ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.11.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.11.1/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.10.x or v1.11.0 to v1.11.1.** > [!IMPORTANT] **Users on v1.11.0 who experienced the memory leaks of longhorn-instance-manager pods [12575](https://github.com/longhorn/longhorn/issues/12575) are highly encouraged to upgrade to v1.11.1 to receive the permanent fix for the proxy connection leaks.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.11.1/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues in this release ### Improvement - [BACKPORT][v1.11.1][IMPROVEMENT] Ensure V2 Engine ReplicaAdd respects the fast-replica-rebuild-enabled setting [12751](https://github.com/longhorn/longhorn/issues/12751) - @davidcheng0922 @roger-ryao - [BACKPORT][v1.11.1][IMPROVEMENT] Topology-aware PV nodeAffinity control: allowedTopologies keys + strictTopology [12689](https://github.com/longhorn/longhorn/issues/12689) - @hookak @roger-ryao - [BACKPORT][v1.11.1][IMPROVEMENT] detailed log for the reason of node controller deleting backing image copies [12585](https://github.com/longhorn/longhorn/issues/12585) - @COLDTURNIP @yangchiu - [BACKPORT][v1.11.1][IMPROVEMENT] Relax `endpoint-network-for-rwx-volume` validation for migratable block-mode volumes [12711](https://github.com/longhorn/longhorn/issues/12711) - @c3y1huang @chriscchien - [BACKPORT][v1.11.1][IMPROVEMENT] RBAC permissions for csi-resizer [12694](https://github.com/longhorn/longhorn/issues/12694) - @yangchiu ### Bug - [BACKPORT][v1.11.1][BUG] Failed replicas accumulate during engine upgrade [12768](https://github.com/longhorn/longhorn/issues/12768) - @davidcheng0922 - [BACKPORT][v1.11.1][BUG] V2 Volume Clone Status is Changed Over Time [12748](https://github.com/longhorn/longhorn/issues/12748) - @davidcheng0922 @roger-ryao - [BACKPORT][v1.11.1][BUG] Backup to S3 fails at 95% [12714](https://github.com/longhorn/longhorn/issues/12714) - @yangchiu @mantissahz - [BACKPORT][v1.11.1][BUG] `spdk_tgt` encountered an assertion failure in `longhorn-spdk-helper` during a CI test run [12738](https://github.com/longhorn/longhorn/issues/12738) - @derekbit @roger-ryao - [BACKPORT][v1.11.1][BUG] Google Cloud Storage (GCS) backup target always fails with SignatureDoesNotMatch due to AWS SDK Go v2 CRC32 checksum incompatibility [12688](https://github.com/longhorn/longhorn/issues/12688) - @mantissahz @chriscchien - [BACKPORT][v1.11.1][BUG] Enable to set defaultSettings.nodeDiskHealthMonitoring [12730](https://github.com/longhorn/longhorn/issues/12730) - @chriscchien - [BACKPORT][v1.11.1][BUG] stale name variable in nsmounter get_pid [12704](https://github.com/longhorn/longhorn/issues/12704) - @chriscchien - [BACKPORT][v1.11.1][BUG] After upgrading to 1.11.0, new persistent volumes have nodeAffinity [12665](https://github.com/longhorn/longhorn/issues/12665) - @chriscchien - [BACKPORT][v1.11.1][BUG] Incorrect storage double-counting causes scheduling failure when multiple replicas exist on the same node [12661](https://github.com/longhorn/longhorn/issues/12661) - @yangchiu @davidcheng0922 - [BACKPORT][v1.11.1][BUG] Recreated block disk with same name never becomes schedulable after volume and disk deletion [12641](https://github.com/longhorn/longhorn/issues/12641) - @davidcheng0922 - [BACKPORT][v1.11.1][BUG] Longhorn v1.10 Volume API is not compatible with the v1.8.1 manifest [12618](https://github.com/longhorn/longhorn/issues/12618) - @mantissahz @roger-ryao - [BACKPORT][v1.11.1][BUG] [v2] Can't use partition as block device [12626](https://github.com/longhorn/longhorn/issues/12626) - @bachmanity1 - [BACKPORT][v1.11.1][BUG] Volume.Spec.CloneMode is empty after upgrading to v1.10.x and following version [12615](https://github.com/longhorn/longhorn/issues/12615) - @mantissahz - [BACKPORT][v1.11.1][BUG] Longhorn validating webhook blocks k3s server node joins - flannel CNI fails to initialize [12589](https://github.com/longhorn/longhorn/issues/12589) - @yangchiu @mantissahz - [BACKPORT][v1.11.1][BUG] V1.11.0 very high memory consumption for instance manager [12575](https://github.com/longhorn/longhorn/issues/12575) - @derekbit @roger-ryao - [BACKPORT][v1.11.1][BUG] Backing image data source pod fails when HTTP proxy is enabled [12780](https://github.com/longhorn/longhorn/issues/12780) - @c3y1huang @chriscchien - [BACKPORT][v1.11.1][BUG] orphan controller does not cleanup the instance on the corresponding instance manager on a multiple IM node [12788](https://github.com/longhorn/longhorn/issues/12788) - @COLDTURNIP @roger-ryao ### Stability - [BACKPORT][v1.11.1][BUG] Potential NEP in Volume Metrics Collector [12733](https://github.com/longhorn/longhorn/issues/12733) - @derekbit @chriscchien ## Contributors - @COLDTURNIP - @PhanLe1010 - @bachmanity1 - @c3y1huang - @chriscchien - @davidcheng0922 - @derekbit - @forbesguthrie - @github-actions[bot] - @hookak - @houhoucoop - @innobead - @mantissahz - @rebeccazzzz - @roger-ryao - @shuo-wu - @sushant-suse - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.4.0.md ================================================ ## Release Note **v1.4.0 released!** 🎆 This release introduces many enhancements, improvements, and bug fixes as described below about stability, performance, data integrity, troubleshooting, and so on. Please try it and feedback. Thanks for all the contributions! - [Kubernetes 1.25 Support](https://github.com/longhorn/longhorn/issues/4003) [[doc]](https://longhorn.io/docs/1.4.0/deploy/important-notes/#pod-security-policies-disabled--pod-security-admission-introduction) In the previous versions, Longhorn relies on Pod Security Policy (PSP) to authorize Longhorn components for privileged operations. From Kubernetes 1.25, PSP has been removed and replaced with Pod Security Admission (PSA). Longhorn v1.4.0 supports opt-in PSP enablement, so it can support Kubernetes versions with or without PSP. - [ARM64 GA](https://github.com/longhorn/longhorn/issues/4206) ARM64 has been experimental from Longhorn v1.1.0. After receiving more user feedback and increasing testing coverage, ARM64 distribution has been stabilized with quality as per our regular regression testing, so it is qualified for general availability. - [RWX GA](https://github.com/longhorn/longhorn/issues/2293) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20220727-dedicated-recovery-backend-for-rwx-volume-nfs-server.md)[[doc]](https://longhorn.io/docs/1.4.0/advanced-resources/rwx-workloads/) RWX has been experimental from Longhorn v1.1.0, but it lacks availability support when the Longhorn Share Manager component behind becomes unavailable. Longhorn v1.4.0 supports NFS recovery backend based on Kubernetes built-in resource, ConfigMap, for recovering NFS client connection during the fail-over period. Also, the NFS client hard mode introduction will further avoid previous potential data loss. For the detail, please check the issue and enhancement proposal. - [Volume Snapshot Checksum](https://github.com/longhorn/longhorn/issues/4210) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20220922-snapshot-checksum-and-bit-rot-detection.md)[[doc]](https://longhorn.io/docs/1.4.0/references/settings/#snapshot-data-integrity) Data integrity is a continuous effort for Longhorn. In this version, Snapshot Checksum has been introduced w/ some settings to allow users to enable or disable checksum calculation with different modes. - [Volume Bit-rot Protection](https://github.com/longhorn/longhorn/issues/3198) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20220922-snapshot-checksum-and-bit-rot-detection.md)[[doc]](https://longhorn.io/docs/1.4.0/references/settings/#snapshot-data-integrity) When enabling the Volume Snapshot Checksum feature, Longhorn will periodically calculate and check the checksums of volume snapshots, find corrupted snapshots, then fix them. - [Volume Replica Rebuilding Speedup](https://github.com/longhorn/longhorn/issues/4783) When enabling the Volume Snapshot Checksum feature, Longhorn will use the calculated snapshot checksum to avoid needless snapshot replication between nodes for improving replica rebuilding speed and resource consumption. - [Volume Trim](https://github.com/longhorn/longhorn/issues/836) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20221103-filesystem-trim.md)[[doc]](https://longhorn.io/docs/1.4.0/volumes-and-nodes/trim-filesystem/#trim-the-filesystem-in-a-longhorn-volume) Longhorn engine supports UNMAP SCSI command to reclaim space from the block volume. - [Online Volume Expansion](https://github.com/longhorn/longhorn/issues/1674) [[doc]](https://longhorn.io/docs/1.4.0/volumes-and-nodes/expansion) Longhorn engine supports optional parameters to pass size expansion requests when updating the volume frontend to support online volume expansion and resize the filesystem via CSI node driver. - [Local Volume via Data Locality Strict Mode](https://github.com/longhorn/longhorn/issues/3957) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20200819-keep-a-local-replica-to-engine.md)[[doc]](https://longhorn.io/docs/1.4.0/references/settings/#default-data-locality) Local volume is based on a new Data Locality setting, Strict Local. It will allow users to create one replica volume staying in a consistent location, and the data transfer between the volume frontend and engine will be through a local socket instead of the TCP stack to improve performance and reduce resource consumption. - [Volume Recurring Job Backup Restore](https://github.com/longhorn/longhorn/issues/2227) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20201002-allow-recurring-backup-detached-volumes.md)[[doc]](https://longhorn.io/docs/1.4.0/snapshots-and-backups/backup-and-restore/restore-recurring-jobs-from-a-backup/) Recurring jobs binding to a volume can be backed up to the remote backup target together with the volume backup metadata. They can be restored back as well for a better operation experience. - [Volume IO Metrics](https://github.com/longhorn/longhorn/issues/2406) [[doc]](https://longhorn.io/docs/1.4.0/monitoring/metrics/#volume) Longhorn enriches Volume metrics by providing real-time IO stats including IOPS, latency, and throughput of R/W IO. Users can set up a monotoning solution like Prometheus to monitor volume performance. - [Longhorn System Backup & Restore](https://github.com/longhorn/longhorn/issues/1455) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20220913-longhorn-system-backup-restore.md)[[doc]](https://longhorn.io/docs/1.4.0/advanced-resources/system-backup-restore/) Users can back up the longhorn system to the remote backup target. Afterward, it's able to restore back to an existing cluster in place or a new cluster for specific operational purposes. - [Support Bundle Enhancement](https://github.com/longhorn/longhorn/issues/2759) [[lep]](https://github.com/longhorn/longhorn/blob/master/enhancements/20221109-support-bundle-enhancement.md) Longhorn introduces a new support bundle integration based on a general [support bundle kit](https://github.com/rancher/support-bundle-kit) solution. This can help us collect more complete troubleshooting info and simulate the cluster environment. - [Tunable Timeout between Engine and Replica](https://github.com/longhorn/longhorn/issues/4491) [[doc]](https://longhorn.io/docs/1.4.0/references/settings/#engine-to-replica-timeout) In the current Longhorn versions, the default timeout between the Longhorn engine and replica is fixed without any exposed user settings. This will potentially bring some challenges for users having a low-spec infra environment. By exporting the setting configurable, it will allow users adaptively tune the stability of volume operations. ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing Longhorn v1.4.0.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.4.0/deploy/install/). ## Upgrade > **Please ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.4.0 from v1.3.x. Only support upgrading from 1.3.x.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.4.0/deploy/upgrade/). ## Deprecation & Incompatibilities - Pod Security Policy is an opt-in setting. If installing Longhorn with PSP support, need to enable it first. - The built-in CSI Snapshotter sidecar is upgraded to v5.0.1. The v1beta1 version of Volume Snapshot custom resource is deprecated but still supported. However, it will be removed after upgrading CSI Snapshotter to 6.1 or later versions in the future, so please start using v1 version instead before the deprecated version is removed. ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Highlights - [FEATURE] Reclaim/Shrink space of volume ([836](https://github.com/longhorn/longhorn/issues/836)) - @yangchiu @derekbit @smallteeths @shuo-wu - [FEATURE] Backup/Restore Longhorn System ([1455](https://github.com/longhorn/longhorn/issues/1455)) - @c3y1huang @khushboo-rancher - [FEATURE] Online volume expansion ([1674](https://github.com/longhorn/longhorn/issues/1674)) - @shuo-wu @chriscchien - [FEATURE] Record recurring schedule in the backups and allow user choose to use it for the restored volume ([2227](https://github.com/longhorn/longhorn/issues/2227)) - @yangchiu @mantissahz - [FEATURE] NFS support (RWX) GA ([2293](https://github.com/longhorn/longhorn/issues/2293)) - @derekbit @chriscchien - [FEATURE] Support metrics for Volume IOPS, throughput and latency real time ([2406](https://github.com/longhorn/longhorn/issues/2406)) - @derekbit @roger-ryao - [FEATURE] Support bundle enhancement ([2759](https://github.com/longhorn/longhorn/issues/2759)) - @c3y1huang @chriscchien - [FEATURE] Automatic identifying of corrupted replica (bit rot detection) ([3198](https://github.com/longhorn/longhorn/issues/3198)) - @yangchiu @derekbit - [FEATURE] Local volume for distributed data workloads ([3957](https://github.com/longhorn/longhorn/issues/3957)) - @derekbit @chriscchien - [IMPROVEMENT] Support K8s 1.25 by updating removed deprecated resource versions like PodSecurityPolicy ([4003](https://github.com/longhorn/longhorn/issues/4003)) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Faster resync time for fresh replica rebuilding ([4092](https://github.com/longhorn/longhorn/issues/4092)) - @yangchiu @derekbit - [FEATURE] Introduce checksum for snapshots ([4210](https://github.com/longhorn/longhorn/issues/4210)) - @derekbit @roger-ryao - [FEATURE] Update K8s version support and component/pkg/build dependencies ([4239](https://github.com/longhorn/longhorn/issues/4239)) - @yangchiu @PhanLe1010 - [BUG] data corruption due to COW and block size not being aligned during rebuilding replicas ([4354](https://github.com/longhorn/longhorn/issues/4354)) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Adjust the iSCSI timeout and the engine-to-replica timeout settings ([4491](https://github.com/longhorn/longhorn/issues/4491)) - @yangchiu @derekbit - [IMPROVEMENT] Using specific block size in Longhorn volume's filesystem ([4594](https://github.com/longhorn/longhorn/issues/4594)) - @derekbit @roger-ryao - [IMPROVEMENT] Speed up replica rebuilding by the metadata such as ctime of snapshot disk files ([4783](https://github.com/longhorn/longhorn/issues/4783)) - @yangchiu @derekbit ## Enhancements - [FEATURE] Configure successfulJobsHistoryLimit of CronJobs ([1711](https://github.com/longhorn/longhorn/issues/1711)) - @weizhe0422 @chriscchien - [FEATURE] Allow customization of the cipher used by cryptsetup in volume encryption ([3353](https://github.com/longhorn/longhorn/issues/3353)) - @mantissahz @chriscchien - [FEATURE] New setting to limit the concurrent volume restoring from backup ([4558](https://github.com/longhorn/longhorn/issues/4558)) - @c3y1huang @chriscchien - [FEATURE] Make FS format options configurable in storage class ([4642](https://github.com/longhorn/longhorn/issues/4642)) - @weizhe0422 @chriscchien ## Improvement - [IMPROVEMENT] Change the script into a docker run command mentioned in 'recovery from longhorn backup without system installed' doc ([1521](https://github.com/longhorn/longhorn/issues/1521)) - @weizhe0422 @chriscchien - [IMPROVEMENT] Improve 'recovery from longhorn backup without system installed' doc. ([1522](https://github.com/longhorn/longhorn/issues/1522)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Dump NFS ganesha logs to pod stdout ([2380](https://github.com/longhorn/longhorn/issues/2380)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Support failed/obsolete orphaned backup cleanup ([3898](https://github.com/longhorn/longhorn/issues/3898)) - @mantissahz @chriscchien - [IMPROVEMENT] liveness and readiness probes with longhorn csi plugin daemonset ([3907](https://github.com/longhorn/longhorn/issues/3907)) - @c3y1huang @roger-ryao - [IMPROVEMENT] Longhorn doesn't reuse failed replica on a disk with full allocated space ([3921](https://github.com/longhorn/longhorn/issues/3921)) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Reduce syscalls while reading and writing requests in longhorn-engine (engine <-> replica) ([4122](https://github.com/longhorn/longhorn/issues/4122)) - @yangchiu @derekbit - [IMPROVEMENT] Reduce read and write calls in liblonghorn (tgt <-> engine) ([4133](https://github.com/longhorn/longhorn/issues/4133)) - @derekbit - [IMPROVEMENT] Replace the GCC allocator in liblonghorn with a more efficient memory allocator ([4136](https://github.com/longhorn/longhorn/issues/4136)) - @yangchiu @derekbit - [DOC] Update Helm readme and document ([4175](https://github.com/longhorn/longhorn/issues/4175)) - @derekbit - [IMPROVEMENT] Purging a volume before rebuilding starts ([4183](https://github.com/longhorn/longhorn/issues/4183)) - @yangchiu @shuo-wu - [IMPROVEMENT] Schedule volumes based on available disk space ([4185](https://github.com/longhorn/longhorn/issues/4185)) - @yangchiu @c3y1huang - [IMPROVEMENT] Recognize default toleration and node selector to allow Longhorn run on the RKE mixed cluster ([4246](https://github.com/longhorn/longhorn/issues/4246)) - @c3y1huang @chriscchien - [IMPROVEMENT] Support bundle doesn't collect the snapshot yamls ([4285](https://github.com/longhorn/longhorn/issues/4285)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Avoid accidentally deleting engine images that are still in use ([4332](https://github.com/longhorn/longhorn/issues/4332)) - @derekbit @chriscchien - [IMPROVEMENT] Show non-JSON error from backup store ([4336](https://github.com/longhorn/longhorn/issues/4336)) - @c3y1huang - [IMPROVEMENT] Update nfs-ganesha to v4.0 ([4351](https://github.com/longhorn/longhorn/issues/4351)) - @derekbit - [IMPROVEMENT] show error when failed to init frontend ([4362](https://github.com/longhorn/longhorn/issues/4362)) - @c3y1huang - [IMPROVEMENT] Too many debug-level log messages in engine instance-manager ([4427](https://github.com/longhorn/longhorn/issues/4427)) - @derekbit @chriscchien - [IMPROVEMENT] Add prep work for fixing the corrupted filesystem using fsck in KB ([4440](https://github.com/longhorn/longhorn/issues/4440)) - @derekbit - [IMPROVEMENT] Prevent users from accidentally uninstalling Longhorn ([4509](https://github.com/longhorn/longhorn/issues/4509)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] add possibility to use nodeSelector on the storageClass ([4574](https://github.com/longhorn/longhorn/issues/4574)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Check if node schedulable condition is set before trying to read it ([4581](https://github.com/longhorn/longhorn/issues/4581)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Review/consolidate the sectorSize in replica server, replica volume, and engine ([4599](https://github.com/longhorn/longhorn/issues/4599)) - @yangchiu @derekbit - [IMPROVEMENT] Reorganize longhorn-manager/k8s/patches and auto-generate preserveUnknownFields field ([4600](https://github.com/longhorn/longhorn/issues/4600)) - @yangchiu @derekbit - [IMPROVEMENT] share-manager pod bypasses the kubernetes scheduler ([4789](https://github.com/longhorn/longhorn/issues/4789)) - @joshimoo @chriscchien - [IMPROVEMENT] Unify the format of returned error messages in longhorn-engine ([4828](https://github.com/longhorn/longhorn/issues/4828)) - @derekbit - [IMPROVEMENT] Longhorn system backup/restore UI ([4855](https://github.com/longhorn/longhorn/issues/4855)) - @smallteeths - [IMPROVEMENT] Replace the modTime (mtime) with ctime in snapshot hash ([4934](https://github.com/longhorn/longhorn/issues/4934)) - @derekbit @chriscchien - [BUG] volume is stuck in attaching/detaching loop with error `Failed to init frontend: device...` ([4959](https://github.com/longhorn/longhorn/issues/4959)) - @derekbit @PhanLe1010 @chriscchien - [IMPROVEMENT] Affinity in the longhorn-ui deployment within the helm chart ([4987](https://github.com/longhorn/longhorn/issues/4987)) - @mantissahz @chriscchien - [IMPROVEMENT] Allow users to change volume.spec.snapshotDataIntegrity on UI ([4994](https://github.com/longhorn/longhorn/issues/4994)) - @yangchiu @smallteeths - [IMPROVEMENT] Backup and restore recurring jobs on UI ([5009](https://github.com/longhorn/longhorn/issues/5009)) - @smallteeths @chriscchien - [IMPROVEMENT] Disable `Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly` for RWX volumes ([5017](https://github.com/longhorn/longhorn/issues/5017)) - @derekbit @chriscchien - [IMPROVEMENT] Enable fast replica rebuilding by default ([5023](https://github.com/longhorn/longhorn/issues/5023)) - @derekbit @roger-ryao - [IMPROVEMENT] Upgrade tcmalloc in longhorn-engine ([5050](https://github.com/longhorn/longhorn/issues/5050)) - @derekbit - [IMPROVEMENT] UI show error when backup target is empty for system backup ([5056](https://github.com/longhorn/longhorn/issues/5056)) - @smallteeths @khushboo-rancher - [IMPROVEMENT] System restore job name should be Longhorn prefixed ([5057](https://github.com/longhorn/longhorn/issues/5057)) - @c3y1huang @khushboo-rancher - [BUG] Error in logs while restoring the system backup ([5061](https://github.com/longhorn/longhorn/issues/5061)) - @c3y1huang @chriscchien - [IMPROVEMENT] Add warning message to when deleting the restoring backups ([5065](https://github.com/longhorn/longhorn/issues/5065)) - @smallteeths @khushboo-rancher @roger-ryao - [IMPROVEMENT] Inconsistent name convention across volume backup restore and system backup restore ([5066](https://github.com/longhorn/longhorn/issues/5066)) - @smallteeths @roger-ryao - [IMPROVEMENT] System restore should proceed to restore other volumes if restoring one volume keeps failing for a certain time. ([5086](https://github.com/longhorn/longhorn/issues/5086)) - @c3y1huang @khushboo-rancher @roger-ryao - [IMPROVEMENT] Support customized number of replicas of webhook and recovery-backend ([5087](https://github.com/longhorn/longhorn/issues/5087)) - @derekbit @chriscchien - [IMPROVEMENT] Simplify the page by placing some configuration items in the advanced configuration when creating the volume ([5090](https://github.com/longhorn/longhorn/issues/5090)) - @yangchiu @smallteeths - [IMPROVEMENT] Support replica sync client timeout setting to stabilize replica rebuilding ([5110](https://github.com/longhorn/longhorn/issues/5110)) - @derekbit @chriscchien - [IMPROVEMENT] Set a newly created volume's data integrity from UI to `ignored` rather than `Fast-Check`. ([5126](https://github.com/longhorn/longhorn/issues/5126)) - @yangchiu @smallteeths ## Performance - [BUG] Turn a node down and up, workload takes longer time to come back online in Longhorn v1.2.0 ([2947](https://github.com/longhorn/longhorn/issues/2947)) - @yangchiu @PhanLe1010 - [TASK] RWX volume performance measurement and investigation ([3665](https://github.com/longhorn/longhorn/issues/3665)) - @derekbit - [TASK] Verify spinning disk/HDD via the current e2e regression ([4182](https://github.com/longhorn/longhorn/issues/4182)) - @yangchiu - [BUG] test_csi_snapshot_snap_create_volume_from_snapshot failed when using HDD as Longhorn disks ([4227](https://github.com/longhorn/longhorn/issues/4227)) - @yangchiu @PhanLe1010 - [TASK] Disable tcmalloc in data path because newer tcmalloc version leads to performance drop ([5096](https://github.com/longhorn/longhorn/issues/5096)) - @derekbit @chriscchien ## Stability - [BUG] Longhorn won't fail all replicas if there is no valid backend during the engine starting stage ([1330](https://github.com/longhorn/longhorn/issues/1330)) - @derekbit @roger-ryao - [BUG] Every other backup fails and crashes the volume (Segmentation Fault) ([1768](https://github.com/longhorn/longhorn/issues/1768)) - @olljanat @mantissahz - [BUG] Backend sizes do not match 5368709120 != 10737418240 in the engine initiation phase ([3601](https://github.com/longhorn/longhorn/issues/3601)) - @derekbit @chriscchien - [BUG] Somehow the Rebuilding field inside volume.meta is set to true causing the volume to stuck in attaching/detaching loop ([4212](https://github.com/longhorn/longhorn/issues/4212)) - @yangchiu @derekbit - [BUG] Engine binary cannot be recovered after being removed accidentally ([4380](https://github.com/longhorn/longhorn/issues/4380)) - @yangchiu @c3y1huang - [TASK] Disable tcmalloc in longhorn-engine and longhorn-instance-manager ([5068](https://github.com/longhorn/longhorn/issues/5068)) - @derekbit ## Bugs - [BUG] Removing old instance records after the new IM pod is launched will take 1 minute ([1363](https://github.com/longhorn/longhorn/issues/1363)) - @mantissahz - [BUG] Restoring volume stuck forever if the backup is already deleted. ([1867](https://github.com/longhorn/longhorn/issues/1867)) - @mantissahz @chriscchien - [BUG] Duplicated default instance manager leads to engine/replica cannot be started ([3000](https://github.com/longhorn/longhorn/issues/3000)) - @PhanLe1010 @roger-ryao - [BUG] Restore from backup sometimes failed if having high frequent recurring backup job w/ retention ([3055](https://github.com/longhorn/longhorn/issues/3055)) - @mantissahz @roger-ryao - [BUG] Newly created backup stays in `InProgress` when the volume deleted before backup finished ([3122](https://github.com/longhorn/longhorn/issues/3122)) - @mantissahz @chriscchien - [Bug] Degraded volume generate failed replica make volume unschedulable ([3220](https://github.com/longhorn/longhorn/issues/3220)) - @derekbit @chriscchien - [BUG] The default access mode of a restored RWX volume is RWO ([3444](https://github.com/longhorn/longhorn/issues/3444)) - @weizhe0422 @roger-ryao - [BUG] Replica rebuilding failure with error "Replica must be closed, Can not add in state: open" ([3828](https://github.com/longhorn/longhorn/issues/3828)) - @mantissahz @roger-ryao - [BUG] Max length of volume name not consist between frontend and backend ([3917](https://github.com/longhorn/longhorn/issues/3917)) - @weizhe0422 @roger-ryao - [BUG] Can't delete volumesnapshot if backup removed first ([4107](https://github.com/longhorn/longhorn/issues/4107)) - @weizhe0422 @chriscchien - [BUG] A IM-proxy connection not closed in full regression 1.3 ([4113](https://github.com/longhorn/longhorn/issues/4113)) - @c3y1huang @chriscchien - [BUG] Scale replica warning ([4120](https://github.com/longhorn/longhorn/issues/4120)) - @c3y1huang @chriscchien - [BUG] Wrong nodeOrDiskEvicted collected in node monitor ([4143](https://github.com/longhorn/longhorn/issues/4143)) - @yangchiu @derekbit - [BUG] Misleading log "BUG: replica is running but storage IP is empty" ([4153](https://github.com/longhorn/longhorn/issues/4153)) - @shuo-wu @chriscchien - [BUG] longhorn-manager cannot start while upgrading if the configmap contains volume sensitive settings ([4160](https://github.com/longhorn/longhorn/issues/4160)) - @derekbit @chriscchien - [BUG] Replica stuck in buggy state with status.currentState is error and the spec.desireState is running ([4197](https://github.com/longhorn/longhorn/issues/4197)) - @yangchiu @PhanLe1010 - [BUG] After updating longhorn to version 1.3.0, only 1 node had problems and I can't even delete it ([4213](https://github.com/longhorn/longhorn/issues/4213)) - @derekbit @c3y1huang @chriscchien - [BUG] Unable to use a TTY error when running environment_check.sh ([4216](https://github.com/longhorn/longhorn/issues/4216)) - @flkdnt @chriscchien - [BUG] The last healthy replica may be evicted or removed ([4238](https://github.com/longhorn/longhorn/issues/4238)) - @yangchiu @shuo-wu - [BUG] Volume detaching and attaching repeatedly while creating multiple snapshots with a same id ([4250](https://github.com/longhorn/longhorn/issues/4250)) - @yangchiu @derekbit - [BUG] Backing image is not deleted and recreated correctly ([4256](https://github.com/longhorn/longhorn/issues/4256)) - @shuo-wu @chriscchien - [BUG] longhorn-ui fails to start on RKE2 with cis-1.6 profile for Longhorn v1.3.0 with helm install ([4266](https://github.com/longhorn/longhorn/issues/4266)) - @yangchiu @mantissahz - [BUG] Longhorn volume stuck in deleting state ([4278](https://github.com/longhorn/longhorn/issues/4278)) - @yangchiu @PhanLe1010 - [BUG] the IP address is duplicate when using storage network and the second network is contronllerd by ovs-cni. ([4281](https://github.com/longhorn/longhorn/issues/4281)) - @mantissahz - [BUG] build longhorn-ui image error ([4283](https://github.com/longhorn/longhorn/issues/4283)) - @smallteeths - [BUG] Wrong conditions in the Chart default-setting manifest for Rancher deployed Windows Cluster feature ([4289](https://github.com/longhorn/longhorn/issues/4289)) - @derekbit @chriscchien - [BUG] Volume operations/rebuilding error during eviction ([4294](https://github.com/longhorn/longhorn/issues/4294)) - @yangchiu @shuo-wu - [BUG] longhorn-manager deletes same pod multi times when rebooting ([4302](https://github.com/longhorn/longhorn/issues/4302)) - @mantissahz @w13915984028 - [BUG] test_setting_backing_image_auto_cleanup failed because the backing image file isn't deleted on the corresponding node as expected ([4308](https://github.com/longhorn/longhorn/issues/4308)) - @shuo-wu @chriscchien - [BUG] After automatically force delete terminating pods of deployment on down node, data lost and I/O error ([4384](https://github.com/longhorn/longhorn/issues/4384)) - @yangchiu @derekbit @PhanLe1010 - [BUG] Volume can not attach to node when engine image DaemonSet pods are not fully deployed ([4386](https://github.com/longhorn/longhorn/issues/4386)) - @PhanLe1010 @chriscchien - [BUG] Error/warning during uninstallation of Longhorn v1.3.1 via manifest ([4405](https://github.com/longhorn/longhorn/issues/4405)) - @PhanLe1010 @roger-ryao - [BUG] can't upgrade engine if a volume was created in Longhorn v1.0 and the volume.spec.dataLocality is `""` ([4412](https://github.com/longhorn/longhorn/issues/4412)) - @derekbit @chriscchien - [BUG] Confusing description the label for replica delition ([4430](https://github.com/longhorn/longhorn/issues/4430)) - @yangchiu @smallteeths - [BUG] Update the Longhorn document in Using the Environment Check Script ([4450](https://github.com/longhorn/longhorn/issues/4450)) - @weizhe0422 @roger-ryao - [BUG] Unable to search 1.3.1 doc by algolia ([4457](https://github.com/longhorn/longhorn/issues/4457)) - @mantissahz @roger-ryao - [BUG] Misleading message "The volume is in expansion progress from size 20Gi to 10Gi" if the expansion is invalid ([4475](https://github.com/longhorn/longhorn/issues/4475)) - @yangchiu @smallteeths - [BUG] Flaky case test_autosalvage_with_data_locality_enabled ([4489](https://github.com/longhorn/longhorn/issues/4489)) - @weizhe0422 - [BUG] Continuously rebuild when auto-balance==least-effort and existing node becomes unschedulable ([4502](https://github.com/longhorn/longhorn/issues/4502)) - @yangchiu @c3y1huang - [BUG] Inconsistent system snapshots between replicas after rebuilding ([4513](https://github.com/longhorn/longhorn/issues/4513)) - @derekbit - [BUG] Prometheus metric for backup state (longhorn_backup_state) returns wrong values ([4521](https://github.com/longhorn/longhorn/issues/4521)) - @mantissahz @roger-ryao - [BUG] Longhorn accidentally schedule all replicas onto a worker node even though the setting Replica Node Level Soft Anti-Affinity is currently disabled ([4546](https://github.com/longhorn/longhorn/issues/4546)) - @yangchiu @mantissahz - [BUG] LH continuously reports `invalid customized default setting taint-toleration` ([4554](https://github.com/longhorn/longhorn/issues/4554)) - @weizhe0422 @roger-ryao - [BUG] the values.yaml in the longhorn helm chart contains values not used. ([4601](https://github.com/longhorn/longhorn/issues/4601)) - @weizhe0422 @roger-ryao - [BUG] longhorn-engine integration test test_restore_to_file_with_backing_file failed after upgrade to sles 15.4 ([4632](https://github.com/longhorn/longhorn/issues/4632)) - @mantissahz - [BUG] Can not pull a backup created by another Longhorn system from the remote backup target ([4637](https://github.com/longhorn/longhorn/issues/4637)) - @yangchiu @mantissahz @roger-ryao - [BUG] Fix the share-manager deletion failure if the confimap is not existing ([4648](https://github.com/longhorn/longhorn/issues/4648)) - @derekbit @roger-ryao - [BUG] Updating volume-scheduling-error failure for RWX volumes and expanding volumes ([4654](https://github.com/longhorn/longhorn/issues/4654)) - @derekbit @chriscchien - [BUG] charts/longhorn/questions.yaml include oudated csi-image tags ([4669](https://github.com/longhorn/longhorn/issues/4669)) - @PhanLe1010 @roger-ryao - [BUG] rebuilding the replica failed after upgrading from 1.2.4 to 1.3.2-rc2 ([4705](https://github.com/longhorn/longhorn/issues/4705)) - @derekbit @chriscchien - [BUG] Cannot re-run helm uninstallation if the first one failed and cannot fetch logs of failed uninstallation pod ([4711](https://github.com/longhorn/longhorn/issues/4711)) - @yangchiu @PhanLe1010 @roger-ryao - [BUG] The old instance-manager-r Pods are not deleted after upgrade ([4726](https://github.com/longhorn/longhorn/issues/4726)) - @mantissahz @chriscchien - [BUG] Replica Auto Balance repeatedly delete the local replica and trigger rebuilding ([4761](https://github.com/longhorn/longhorn/issues/4761)) - @c3y1huang @roger-ryao - [BUG] Volume metafile getting deleted or empty results in a detach-attach loop ([4846](https://github.com/longhorn/longhorn/issues/4846)) - @mantissahz @chriscchien - [BUG] Backing image is stuck at `in-progress` status if the provided checksum is incorrect ([4852](https://github.com/longhorn/longhorn/issues/4852)) - @FrankYang0529 @chriscchien - [BUG] Duplicate channel close error in the backing image manage related components ([4865](https://github.com/longhorn/longhorn/issues/4865)) - @weizhe0422 @roger-ryao - [BUG] The node ID of backing image data source somehow get changed then lead to file handling failed ([4887](https://github.com/longhorn/longhorn/issues/4887)) - @shuo-wu @chriscchien - [BUG] Cannot upload a backing image larger than 10G ([4902](https://github.com/longhorn/longhorn/issues/4902)) - @smallteeths @shuo-wu @chriscchien - [BUG] Failed to build longhorn-instance-manager master branch ([4946](https://github.com/longhorn/longhorn/issues/4946)) - @derekbit - [BUG] PVC only works with plural annotation `volumes.kubernetes.io/storage-provisioner: driver.longhorn.io` ([4951](https://github.com/longhorn/longhorn/issues/4951)) - @weizhe0422 - [BUG] Failed to create a replenished replica process because of the newly adding option ([4962](https://github.com/longhorn/longhorn/issues/4962)) - @yangchiu @derekbit - [BUG] Incorrect log messages in longhorn-engine processRemoveSnapshot() ([4980](https://github.com/longhorn/longhorn/issues/4980)) - @derekbit - [BUG] System backup showing wrong age ([5047](https://github.com/longhorn/longhorn/issues/5047)) - @smallteeths @khushboo-rancher - [BUG] System backup should validate empty backup target ([5055](https://github.com/longhorn/longhorn/issues/5055)) - @c3y1huang @khushboo-rancher - [BUG] missing the `restoreVolumeRecurringJob` parameter in the VolumeGet API ([5062](https://github.com/longhorn/longhorn/issues/5062)) - @mantissahz @roger-ryao - [BUG] System restore stuck in restoring if pvc exists with identical name ([5064](https://github.com/longhorn/longhorn/issues/5064)) - @c3y1huang @roger-ryao - [BUG] No error shown on UI if system backup conf not available ([5072](https://github.com/longhorn/longhorn/issues/5072)) - @c3y1huang @khushboo-rancher - [BUG] System restore missing services ([5074](https://github.com/longhorn/longhorn/issues/5074)) - @yangchiu @c3y1huang - [BUG] In a system restore, PV & PVC are not restored if PVC was created with 'longhorn-static' (created via Longhorn GUI) ([5091](https://github.com/longhorn/longhorn/issues/5091)) - @c3y1huang @khushboo-rancher - [BUG][v1.4.0-rc1] image security scan CRITICAL issues ([5107](https://github.com/longhorn/longhorn/issues/5107)) - @yangchiu @mantissahz - [BUG] Snapshot trim wrong label in the volume detail page. ([5127](https://github.com/longhorn/longhorn/issues/5127)) - @smallteeths @chriscchien - [BUG] Filesystem on the volume with a backing image is corrupted after applying trim operation ([5129](https://github.com/longhorn/longhorn/issues/5129)) - @derekbit @chriscchien - [BUG] Error in uninstall job ([5132](https://github.com/longhorn/longhorn/issues/5132)) - @c3y1huang @chriscchien - [BUG] Uninstall job unable to delete the systembackup and systemrestore cr. ([5133](https://github.com/longhorn/longhorn/issues/5133)) - @c3y1huang @chriscchien - [BUG] Nil pointer dereference error on restoring the system backup ([5134](https://github.com/longhorn/longhorn/issues/5134)) - @yangchiu @c3y1huang - [BUG] UI option Update Replicas Auto Balance should use capital letter like others ([5154](https://github.com/longhorn/longhorn/issues/5154)) - @smallteeths @chriscchien - [BUG] System restore cannot roll out when volume name is different to the PV ([5157](https://github.com/longhorn/longhorn/issues/5157)) - @yangchiu @c3y1huang - [BUG] Online expansion doesn't succeed after a failed expansion ([5169](https://github.com/longhorn/longhorn/issues/5169)) - @derekbit @shuo-wu @khushboo-rancher ## Misc - [DOC] RWX support for NVIDIA JETSON Ubuntu 18.4LTS kernel requires enabling NFSV4.1 ([3157](https://github.com/longhorn/longhorn/issues/3157)) - @yangchiu @derekbit - [DOC] Add information about encryption algorithm to documentation ([3285](https://github.com/longhorn/longhorn/issues/3285)) - @mantissahz - [DOC] Update the doc of volume size after introducing snapshot prune ([4158](https://github.com/longhorn/longhorn/issues/4158)) - @shuo-wu - [Doc] Update the outdated "Customizing Default Settings" document ([4174](https://github.com/longhorn/longhorn/issues/4174)) - @derekbit - [TASK] Refresh distro version support for 1.4 ([4401](https://github.com/longhorn/longhorn/issues/4401)) - @weizhe0422 - [TASK] Update official document Longhorn Networking ([4478](https://github.com/longhorn/longhorn/issues/4478)) - @derekbit - [TASK] Update preserveUnknownFields fields in longhorn-manager CRD manifest ([4505](https://github.com/longhorn/longhorn/issues/4505)) - @derekbit @roger-ryao - [TASK] Disable doc search for archived versions < 1.1 ([4524](https://github.com/longhorn/longhorn/issues/4524)) - @mantissahz - [TASK] Update longhorn components with the latest backupstore ([4552](https://github.com/longhorn/longhorn/issues/4552)) - @derekbit - [TASK] Update base image of all components from BCI 15.3 to 15.4 ([4617](https://github.com/longhorn/longhorn/issues/4617)) - @yangchiu - [DOC] Update the Longhorn document in Install with Helm ([4745](https://github.com/longhorn/longhorn/issues/4745)) - @roger-ryao - [TASK] Create longhornio support-bundle-kit image ([4911](https://github.com/longhorn/longhorn/issues/4911)) - @yangchiu - [DOC] Add Recurring * Jobs History Limit to setting reference ([4912](https://github.com/longhorn/longhorn/issues/4912)) - @weizhe0422 @roger-ryao - [DOC] Add Failed Backup TTL to setting reference ([4913](https://github.com/longhorn/longhorn/issues/4913)) - @mantissahz - [TASK] Create longhornio liveness probe image ([4945](https://github.com/longhorn/longhorn/issues/4945)) - @yangchiu - [TASK] Make system managed components branch-based build ([5024](https://github.com/longhorn/longhorn/issues/5024)) - @yangchiu - [TASK] Remove unstable s390x from PR check for all repos ([5040](https://github.com/longhorn/longhorn/issues/5040)) - - [TASK] Update longhorn-share-manager's nfs-ganesha to V4.2.1 ([5083](https://github.com/longhorn/longhorn/issues/5083)) - @derekbit @mantissahz - [DOC] Update the Longhorn document in Setting up Prometheus and Grafana ([5158](https://github.com/longhorn/longhorn/issues/5158)) - @roger-ryao ## Contributors - @FrankYang0529 - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @flkdnt - @innobead - @joshimoo - @khushboo-rancher - @mantissahz - @olljanat - @roger-ryao - @shuo-wu - @smallteeths - @w13915984028 - @weizhe0422 - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.4.1.md ================================================ ## Release Note **v1.4.1 released!** 🎆 This release introduces improvements and bug fixes as described below about stability, performance, space efficiency, resilience, and so on. Please try it and feedback. Thanks for all the contributions! ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing Longhorn v1.4.1.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.4.1/deploy/install/). ## Upgrade > **Please ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.4.1 from v1.3.x/v1.4.0, which are only supported source versions.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.4.1/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Highlights - [IMPROVEMENT] Periodically clean up volume snapshots ([3836](https://github.com/longhorn/longhorn/issues/3836)) - @c3y1huang @chriscchien ## Improvement - [IMPROVEMENT] Do not count the failure replica reuse failure caused by the disconnection ([1923](https://github.com/longhorn/longhorn/issues/1923)) - @yangchiu @mantissahz - [IMPROVEMENT] Update uninstallation info to include the 'Deleting Confirmation Flag' in chart ([5250](https://github.com/longhorn/longhorn/issues/5250)) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Fix Guaranteed Engine Manager CPU recommendation formula in UI ([5338](https://github.com/longhorn/longhorn/issues/5338)) - @c3y1huang @smallteeths @roger-ryao - [IMPROVEMENT] Update PSP validation in the Longhorn upstream chart ([5339](https://github.com/longhorn/longhorn/issues/5339)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Update ganesha nfs to 4.2.3 ([5356](https://github.com/longhorn/longhorn/issues/5356)) - @derekbit @roger-ryao - [IMPROVEMENT] Set write-cache of longhorn block device to off explicitly ([5382](https://github.com/longhorn/longhorn/issues/5382)) - @derekbit @chriscchien ## Stability - [BUG] Memory leak in CSI plugin caused by stuck umount processes if the RWX volume is already gone ([5296](https://github.com/longhorn/longhorn/issues/5296)) - @derekbit @roger-ryao - [BUG] share-manager pod failed to restart after kubelet restart ([5507](https://github.com/longhorn/longhorn/issues/5507)) - @yangchiu @derekbit ## Bugs - [BUG] Longhorn 1.3.2 fails to backup & restore volumes behind Internet proxy ([5054](https://github.com/longhorn/longhorn/issues/5054)) - @mantissahz @chriscchien - [BUG] RWX doesn't work with release 1.4.0 due to end grace update error from recovery backend ([5183](https://github.com/longhorn/longhorn/issues/5183)) - @derekbit @chriscchien - [BUG] Incorrect indentation of charts/questions.yaml ([5196](https://github.com/longhorn/longhorn/issues/5196)) - @mantissahz @roger-ryao - [BUG] Updating option "Allow snapshots removal during trim" for old volumes failed ([5218](https://github.com/longhorn/longhorn/issues/5218)) - @shuo-wu @roger-ryao - [BUG] Incorrect router retry mechanism ([5259](https://github.com/longhorn/longhorn/issues/5259)) - @mantissahz @chriscchien - [BUG] System Backup is stuck at Uploading if there are PVs not provisioned by CSI driver ([5286](https://github.com/longhorn/longhorn/issues/5286)) - @c3y1huang @chriscchien - [BUG] Sync up with backup target during DR volume activation ([5292](https://github.com/longhorn/longhorn/issues/5292)) - @yangchiu @weizhe0422 - [BUG] environment_check.sh does not handle different kernel versions in cluster correctly ([5304](https://github.com/longhorn/longhorn/issues/5304)) - @achims311 @roger-ryao - [BUG] instance-manager-r high memory consumption ([5312](https://github.com/longhorn/longhorn/issues/5312)) - @derekbit @roger-ryao - [BUG] Replica rebuilding caused by rke2/kubelet restart ([5340](https://github.com/longhorn/longhorn/issues/5340)) - @derekbit @chriscchien - [BUG] Error message not consistent between create/update recurring job when retain number greater than 50 ([5434](https://github.com/longhorn/longhorn/issues/5434)) - @c3y1huang @chriscchien - [BUG] Do not copy Host header to API requests forwarded to Longhorn Manager ([5438](https://github.com/longhorn/longhorn/issues/5438)) - @yangchiu @smallteeths - [BUG] RWX Volume attachment is getting Failed ([5456](https://github.com/longhorn/longhorn/issues/5456)) - @derekbit - [BUG] test case test_backup_lock_deletion_during_restoration failed ([5458](https://github.com/longhorn/longhorn/issues/5458)) - @yangchiu @derekbit - [BUG] [master] [v1.4.1-rc1] Volume restoration will never complete if attached node is down ([5464](https://github.com/longhorn/longhorn/issues/5464)) - @derekbit @weizhe0422 @chriscchien - [BUG] Unable to create support bundle agent pod in air-gap environment ([5467](https://github.com/longhorn/longhorn/issues/5467)) - @yangchiu @c3y1huang - [BUG] Node disconnection test failed ([5476](https://github.com/longhorn/longhorn/issues/5476)) - @yangchiu @derekbit - [BUG] Physical node down test failed ([5477](https://github.com/longhorn/longhorn/issues/5477)) - @derekbit @chriscchien - [BUG] Backing image with sync failure ([5481](https://github.com/longhorn/longhorn/issues/5481)) - @ChanYiLin @roger-ryao - [BUG] Example of data migration doesn't work for hidden/./dot-files) ([5484](https://github.com/longhorn/longhorn/issues/5484)) - @hedefalk @shuo-wu @chriscchien - [BUG] test case test_dr_volume_with_backup_block_deletion failed ([5489](https://github.com/longhorn/longhorn/issues/5489)) - @yangchiu @derekbit ## Misc - [TASK][UI] add new recurring job tasks ([5272](https://github.com/longhorn/longhorn/issues/5272)) - @smallteeths @chriscchien ## Contributors - @ChanYiLin - @PhanLe1010 - @achims311 - @c3y1huang - @chriscchien - @derekbit - @hedefalk - @innobead - @mantissahz - @roger-ryao - @shuo-wu - @smallteeths - @weizhe0422 - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.4.2.md ================================================ ## Release Note ### **v1.4.2 released!** 🎆 Longhorn v1.4.2 is the latest stable version of Longhorn 1.4. It introduces improvements and bug fixes in the areas of stability, performance, space efficiency, resilience, and so on. Please try it out and provide feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing v1.4.2.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.4.2/deploy/install/). ## Upgrade > **Please read the [important notes](https://longhorn.io/docs/1.4.2/deploy/important-notes/) first and ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.4.2 from v1.3.x/v1.4.x, which are only supported source versions.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.4.2/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Highlights - [IMPROVEMENT] Use PDB to protect Longhorn components from unexpected drains ([3304](https://github.com/longhorn/longhorn/issues/3304)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Introduce timeout mechanism for the sparse file syncing service ([4305](https://github.com/longhorn/longhorn/issues/4305)) - @yangchiu @ChanYiLin - [IMPROVEMENT] Recurring jobs create new snapshots while being not able to clean up old ones ([4898](https://github.com/longhorn/longhorn/issues/4898)) - @mantissahz @chriscchien ## Improvement - [IMPROVEMENT] Support bundle collects dmesg, syslog and related information of longhorn nodes ([5073](https://github.com/longhorn/longhorn/issues/5073)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Fix BackingImage uploading/downloading flow to prevent client timeout ([5443](https://github.com/longhorn/longhorn/issues/5443)) - @ChanYiLin @chriscchien - [IMPROVEMENT] Create a new setting so that Longhorn removes PDB for instance-manager-r that doesn't have any running instance inside it ([5549](https://github.com/longhorn/longhorn/issues/5549)) - @PhanLe1010 @khushboo-rancher - [IMPROVEMENT] Deprecate the setting `allow-node-drain-with-last-healthy-replica` and replace it by `node-drain-policy` setting ([5585](https://github.com/longhorn/longhorn/issues/5585)) - @yangchiu @PhanLe1010 - [IMPROVEMENT][UI] Recurring jobs create new snapshots while being not able to clean up old one ([5610](https://github.com/longhorn/longhorn/issues/5610)) - @mantissahz @smallteeths @roger-ryao - [IMPROVEMENT] Only activate replica if it doesn't have deletion timestamp during volume engine upgrade ([5632](https://github.com/longhorn/longhorn/issues/5632)) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Clean up backup target if the backup target setting is unset ([5655](https://github.com/longhorn/longhorn/issues/5655)) - @yangchiu @ChanYiLin ## Resilience - [BUG] Directly mark replica as failed if the node is deleted ([5542](https://github.com/longhorn/longhorn/issues/5542)) - @weizhe0422 @roger-ryao - [BUG] RWX volume is stuck at detaching when the attached node is down ([5558](https://github.com/longhorn/longhorn/issues/5558)) - @derekbit @roger-ryao - [BUG] Backup monitor gets stuck in an infinite loop if backup isn't found ([5662](https://github.com/longhorn/longhorn/issues/5662)) - @derekbit @chriscchien - [BUG] Resources such as replicas are somehow not mutated when network is unstable ([5762](https://github.com/longhorn/longhorn/issues/5762)) - @derekbit @roger-ryao - [BUG] Instance manager may not update instance status for a minute after starting ([5809](https://github.com/longhorn/longhorn/issues/5809)) - @ejweber @chriscchien ## Bugs - [BUG] Delete a uploading backing image, the corresponding LH temp file is not deleted ([3682](https://github.com/longhorn/longhorn/issues/3682)) - @ChanYiLin @chriscchien - [BUG] Can not create backup in engine image not fully deployed cluster ([5248](https://github.com/longhorn/longhorn/issues/5248)) - @ChanYiLin @roger-ryao - [BUG] Upgrade engine --> spec.restoreVolumeRecurringJob and spec.snapshotDataIntegrity Unsupported value ([5485](https://github.com/longhorn/longhorn/issues/5485)) - @yangchiu @derekbit - [BUG] Bulk backup deletion cause restoring volume to finish with attached state. ([5506](https://github.com/longhorn/longhorn/issues/5506)) - @ChanYiLin @roger-ryao - [BUG] volume expansion starts for no reason, gets stuck on current size > expected size ([5513](https://github.com/longhorn/longhorn/issues/5513)) - @mantissahz @roger-ryao - [BUG] RWX volume attachment failed if tried more enough times ([5537](https://github.com/longhorn/longhorn/issues/5537)) - @yangchiu @derekbit - [BUG] instance-manager-e emits `Wait for process pvc-xxxx to shutdown` constantly ([5575](https://github.com/longhorn/longhorn/issues/5575)) - @derekbit @roger-ryao - [BUG] Support bundle kit should respect node selector & taint toleration ([5614](https://github.com/longhorn/longhorn/issues/5614)) - @yangchiu @c3y1huang - [BUG] Value overlapped in page Instance Manager Image ([5622](https://github.com/longhorn/longhorn/issues/5622)) - @smallteeths @chriscchien - [BUG] Instance manager PDB created with wrong selector thus blocking the draining of the wrongly selected node forever ([5680](https://github.com/longhorn/longhorn/issues/5680)) - @PhanLe1010 @chriscchien - [BUG] During volume live engine upgrade, if the replica pod is killed, the volume is stuck in upgrading forever ([5684](https://github.com/longhorn/longhorn/issues/5684)) - @yangchiu @PhanLe1010 - [BUG] Instance manager PDBs cannot be removed if the longhorn-manager pod on its spec node is not available ([5688](https://github.com/longhorn/longhorn/issues/5688)) - @PhanLe1010 @roger-ryao - [BUG] Rebuild rebuilding is possibly issued to a wrong replica ([5709](https://github.com/longhorn/longhorn/issues/5709)) - @ejweber @roger-ryao - [BUG] longhorn upgrade is not upgrading engineimage ([5740](https://github.com/longhorn/longhorn/issues/5740)) - @shuo-wu @chriscchien - [BUG] `test_replica_auto_balance_when_replica_on_unschedulable_node` Error in creating volume with nodeSelector and dataLocality parameters ([5745](https://github.com/longhorn/longhorn/issues/5745)) - @c3y1huang @roger-ryao - [BUG] Unable to backup volume after NFS server IP change ([5856](https://github.com/longhorn/longhorn/issues/5856)) - @derekbit @roger-ryao ## Misc - [TASK] Check and update the networking doc & example YAMLs ([5651](https://github.com/longhorn/longhorn/issues/5651)) - @yangchiu @shuo-wu ## Contributors - @ChanYiLin - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @khushboo-rancher - @mantissahz - @roger-ryao - @shuo-wu - @smallteeths - @weizhe0422 - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.4.3.md ================================================ ## Release Note ### **v1.4.3 released!** 🎆 Longhorn v1.4.3 is the latest stable version of Longhorn 1.4. It introduces improvements and bug fixes in the areas of stability, resilience, and so on. Please try it out and provide feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing v1.4.3.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.4.3/deploy/install/). ## Upgrade > **Please read the [important notes](https://longhorn.io/docs/1.4.3/deploy/important-notes/) first and ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.4.3 from v1.3.x/v1.4.x, which are only supported source versions.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.4.3/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Improvement - [IMPROVEMENT] Assign the pods to the same node where the strict-local volume is present ([5448](https://github.com/longhorn/longhorn/issues/5448)) - @c3y1huang @chriscchien ## Resilience - [BUG] filesystem corrupted after delete instance-manager-r for a locality best-effort volume ([5801](https://github.com/longhorn/longhorn/issues/5801)) - @yangchiu @ChanYiLin @mantissahz ## Bugs - [BUG] 'Upgrade Engine' still shows up in a specific situation when engine already upgraded ([3063](https://github.com/longhorn/longhorn/issues/3063)) - @weizhe0422 @PhanLe1010 @smallteeths - [BUG] DR volume even after activation remains in standby mode if there are one or more failed replicas. ([3069](https://github.com/longhorn/longhorn/issues/3069)) - @yangchiu @mantissahz - [BUG] Prevent Longhorn uninstallation from getting stuck due to backups in error ([5868](https://github.com/longhorn/longhorn/issues/5868)) - @ChanYiLin @mantissahz - [BUG] Unable to create support bundle if the previous one stayed in ReadyForDownload phase ([5882](https://github.com/longhorn/longhorn/issues/5882)) - @c3y1huang @roger-ryao - [BUG] share-manager for a given pvc keep restarting (other pvc are working fine) ([5954](https://github.com/longhorn/longhorn/issues/5954)) - @yangchiu @derekbit - [BUG] Replica auto-rebalance doesn't respect node selector ([5971](https://github.com/longhorn/longhorn/issues/5971)) - @c3y1huang @roger-ryao - [BUG] Extra snapshot generated when clone from a detached volume ([5986](https://github.com/longhorn/longhorn/issues/5986)) - @weizhe0422 @ejweber - [BUG] User created snapshot deleted after node drain and uncordon ([5992](https://github.com/longhorn/longhorn/issues/5992)) - @yangchiu @mantissahz - [BUG] In some specific situation, system backup auto deleted when creating another one ([6045](https://github.com/longhorn/longhorn/issues/6045)) - @c3y1huang @chriscchien - [BUG] Backing Image deletion stuck if it's deleted during uploading process and bids is ready-for-transfer state ([6086](https://github.com/longhorn/longhorn/issues/6086)) - @WebberHuang1118 @chriscchien - [BUG] Backing image manager fails when SELinux is enabled ([6108](https://github.com/longhorn/longhorn/issues/6108)) - @ejweber @chriscchien - [BUG] test_dr_volume_with_restore_command_error failed ([6130](https://github.com/longhorn/longhorn/issues/6130)) - @mantissahz @roger-ryao - [BUG] Longhorn doesn't remove the system backups crd on uninstallation ([6185](https://github.com/longhorn/longhorn/issues/6185)) - @c3y1huang @khushboo-rancher - [BUG] Test case test_ha_backup_deletion_recovery failed in rhel or rockylinux arm64 environment ([6213](https://github.com/longhorn/longhorn/issues/6213)) - @yangchiu @ChanYiLin @mantissahz - [BUG] Engine continues to attempt to rebuild replica while detaching ([6217](https://github.com/longhorn/longhorn/issues/6217)) - @yangchiu @ejweber - [BUG] Unable to receive support bundle from UI when it's large (400MB+) ([6256](https://github.com/longhorn/longhorn/issues/6256)) - @c3y1huang @chriscchien - [BUG] Migration test case failed: unable to detach volume migration is not ready yet ([6238](https://github.com/longhorn/longhorn/issues/6238)) - @yangchiu @PhanLe1010 @khushboo-rancher - [BUG] Restored Volumes stuck in attaching state ([6239](https://github.com/longhorn/longhorn/issues/6239)) - @derekbit @roger-ryao ## Contributors - @ChanYiLin - @PhanLe1010 - @WebberHuang1118 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @khushboo-rancher - @mantissahz - @roger-ryao - @smallteeths - @weizhe0422 - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.0.md ================================================ ## Release Note ### **v1.5.0 released!** 🎆 Longhorn v1.5.0 is the latest version of Longhorn 1.5. It introduces many enhancements, improvements, and bug fixes as described below including performance, stability, maintenance, resilience, and so on. Please try it and feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). - [v2 Data Engine based on SPDK - Preview](https://github.com/longhorn/longhorn/issues/5751) > **Please note that this is a preview feature, so should not be used in any production environment. A preview feature is disabled by default and would be changed in the following versions until it becomes general availability.** In addition to the existing iSCSI stack (v1) data engine, we are introducing the v2 data engine based on SPDK (Storage Performance Development Kit). This release includes the introduction of volume lifecycle management, degraded volume handling, offline replica rebuilding, block device management, and orphaned replica management. For the performance benchmark and comparison with v1, check the report [here](https://longhorn.io/docs/1.5.0/spdk/performance-benchmark/). - [Longhorn Volume Attachment](https://github.com/longhorn/longhorn/issues/3715) Introducing the new Longhorn VolumeAttachment CR, which ensures exclusive attachment and supports automatic volume attachment and detachment for various headless operations such as volume cloning, backing image export, and recurring jobs. - [Cluster Autoscaler - GA](https://github.com/longhorn/longhorn/issues/5238) Cluster Autoscaler was initially introduced as an experimental feature in v1.3. After undergoing automatic validation on different public cloud Kubernetes distributions and receiving user feedback, it has now reached general availability. - [Instance Manager Engine & Replica Consolidation](https://github.com/longhorn/longhorn/issues/5208) Previously, there were two separate instance manager pods responsible for volume engine and replica process management. However, this setup required high resource usage, especially during live upgrades. In this release, we have merged these pods into a single instance manager, reducing the initial resource requirements. - [Volume Backup Compression Methods](https://github.com/longhorn/longhorn/issues/5189) Longhorn supports different compression methods for volume backups, including lz4, gzip, or no compression. This allows users to choose the most suitable method based on their data type and usage requirements. - [Automatic Volume Trim Recurring Job](https://github.com/longhorn/longhorn/issues/5186) While volume filesystem trim was introduced in v1.4, users had to perform the operation manually. From this release, users can create a recurring job that automatically runs the trim process, improving space efficiency without requiring human intervention. - [RWX Volume Trim](https://github.com/longhorn/longhorn/issues/5143) Longhorn supports filesystem trim for RWX (Read-Write-Many) volumes, expanding the trim functionality beyond RWO (Read-Write-Once) volumes only. - [Upgrade Path Enforcement & Downgrade Prevention](https://github.com/longhorn/longhorn/issues/5131) To ensure compatibility after an upgrade, we have implemented upgrade path enforcement. This prevents unintended downgrades and ensures the system and data remain intact. - [Backing Image Management via CSI VolumeSnapshot](https://github.com/longhorn/longhorn/issues/5005) Users can now utilize the unified CSI VolumeSnapshot interface to manage Backing Images similar to volume snapshots and backups. - [Snapshot Cleanup & Delete Recurring Job](https://github.com/longhorn/longhorn/issues/3836) Introducing two new recurring job types specifically designed for snapshot cleanup and deletion. These jobs allow users to remove unnecessary snapshots for better space efficiency. - [CIFS Backup Store](https://github.com/longhorn/longhorn/issues/3599) & [Azure Backup Store](https://github.com/longhorn/longhorn/issues/1309) To enhance users' backup strategies and align with data governance policies, Longhorn now supports additional backup storage protocols, including CIFS and Azure. - [Kubernetes Upgrade Node Drain Policy](https://github.com/longhorn/longhorn/issues/3304) The new Node Drain Policy provides flexible strategies to protect volume data during Kubernetes upgrades or node maintenance operations. This ensures the integrity and availability of your volumes. ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing Longhorn v1.5.0.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.5.0/deploy/install/). ## Upgrade > **Please ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.5.0 from v1.4.x. Only support upgrading from 1.4.x.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.5.0/deploy/upgrade/). ## Deprecation & Incompatibilities Please check the [important notes](https://longhorn.io/docs/1.5.0/deploy/important-notes/) to know more about deprecated, removed, incompatible features and important changes. If you upgrade indirectly from an older version like v1.3.x, please also check the corresponding important note for each upgrade version path. ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Highlights - [DOC] Provide the user guide for Kubernetes upgrade ([494](https://github.com/longhorn/longhorn/issues/494)) - @PhanLe1010 - [FEATURE] Backups to Azure Blob Storage ([1309](https://github.com/longhorn/longhorn/issues/1309)) - @mantissahz @chriscchien - [IMPROVEMENT] Use PDB to protect Longhorn components from unexpected drains ([3304](https://github.com/longhorn/longhorn/issues/3304)) - @yangchiu @PhanLe1010 - [FEATURE] CIFS Backup Store Support ([3599](https://github.com/longhorn/longhorn/issues/3599)) - @derekbit @chriscchien - [IMPROVEMENT] Consolidate volume attach/detach implementation ([3715](https://github.com/longhorn/longhorn/issues/3715)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Periodically clean up volume snapshots ([3836](https://github.com/longhorn/longhorn/issues/3836)) - @c3y1huang @chriscchien - [IMPROVEMENT] Introduce timeout mechanism for the sparse file syncing service ([4305](https://github.com/longhorn/longhorn/issues/4305)) - @yangchiu @ChanYiLin - [IMPROVEMENT] Recurring jobs create new snapshots while being not able to clean up old ones ([4898](https://github.com/longhorn/longhorn/issues/4898)) - @mantissahz @chriscchien - [FEATURE] BackingImage Management via VolumeSnapshot ([5005](https://github.com/longhorn/longhorn/issues/5005)) - @ChanYiLin @chriscchien - [FEATURE] Upgrade path enforcement & downgrade prevention ([5131](https://github.com/longhorn/longhorn/issues/5131)) - @yangchiu @mantissahz - [FEATURE] Support RWX volume trim ([5143](https://github.com/longhorn/longhorn/issues/5143)) - @derekbit @chriscchien - [FEATURE] Auto Trim via recurring job ([5186](https://github.com/longhorn/longhorn/issues/5186)) - @c3y1huang @chriscchien - [FEATURE] Introduce faster compression and multiple threads for volume backup & restore ([5189](https://github.com/longhorn/longhorn/issues/5189)) - @derekbit @roger-ryao - [FEATURE] Consolidate Instance Manager Engine & Replica for resource consumption reduction ([5208](https://github.com/longhorn/longhorn/issues/5208)) - @yangchiu @c3y1huang - [FEATURE] Cluster Autoscaler Support GA ([5238](https://github.com/longhorn/longhorn/issues/5238)) - @yangchiu @c3y1huang - [FEATURE] Update K8s version support and component/pkg/build dependencies for Longhorn 1.5 ([5595](https://github.com/longhorn/longhorn/issues/5595)) - @yangchiu @ejweber - [FEATURE] Support SPDK Data Engine - Preview ([5751](https://github.com/longhorn/longhorn/issues/5751)) - @derekbit @shuo-wu @DamiaSan ## Enhancements - [FEATURE] Allow users to directly activate a restoring/DR volume as long as there is one ready replica. ([1512](https://github.com/longhorn/longhorn/issues/1512)) - @mantissahz @weizhe0422 - [REFACTOR] volume controller refactoring/split up, to simplify the control flow ([2527](https://github.com/longhorn/longhorn/issues/2527)) - @PhanLe1010 @chriscchien - [FEATURE] Import and export SPDK longhorn volumes to longhorn sparse file directory ([4100](https://github.com/longhorn/longhorn/issues/4100)) - @DamiaSan - [FEATURE] Add a global `storage reserved` setting for newly created longhorn nodes' disks ([4773](https://github.com/longhorn/longhorn/issues/4773)) - @mantissahz @chriscchien - [FEATURE] Support backup volumes during system backup ([5011](https://github.com/longhorn/longhorn/issues/5011)) - @c3y1huang @chriscchien - [FEATURE] Support SPDK lvol shallow copy for newly replica creation ([5217](https://github.com/longhorn/longhorn/issues/5217)) - @DamiaSan - [FEATURE] Introduce longhorn-spdk-engine for SPDK volume management ([5282](https://github.com/longhorn/longhorn/issues/5282)) - @shuo-wu - [FEATURE] Support replica-zone-soft-anti-affinity setting per volume ([5358](https://github.com/longhorn/longhorn/issues/5358)) - @ChanYiLin @smallteeths @chriscchien - [FEATURE] Install Opt-In NetworkPolicies ([5403](https://github.com/longhorn/longhorn/issues/5403)) - @yangchiu @ChanYiLin - [FEATURE] Create Longhorn SPDK Engine component with basic fundamental functions ([5406](https://github.com/longhorn/longhorn/issues/5406)) - @shuo-wu - [FEATURE] Add status APIs for shallow copy and IO pause/resume ([5647](https://github.com/longhorn/longhorn/issues/5647)) - @DamiaSan - [FEATURE] Introduce a new disk type, disk management and replica scheduler for SPDK volumes ([5683](https://github.com/longhorn/longhorn/issues/5683)) - @derekbit @roger-ryao - [FEATURE] Support replica scheduling for SPDK volume ([5711](https://github.com/longhorn/longhorn/issues/5711)) - @derekbit - [FEATURE] Create SPDK gRPC service for instance manager ([5712](https://github.com/longhorn/longhorn/issues/5712)) - @shuo-wu - [FEATURE] Environment check script for Longhorn with SPDK ([5738](https://github.com/longhorn/longhorn/issues/5738)) - @derekbit @chriscchien - [FEATURE] Deployment manifests for helping install SPDK dependencies, utilities and libraries ([5739](https://github.com/longhorn/longhorn/issues/5739)) - @yangchiu @derekbit - [FEATURE] Implement Disk gRPC Service in Instance Manager for collecting SPDK disk statistics from SPDK gRPC service ([5744](https://github.com/longhorn/longhorn/issues/5744)) - @derekbit @chriscchien - [FEATURE] Support for SPDK RAID1 by setting the minimum number of base_bdevs to 1 ([5758](https://github.com/longhorn/longhorn/issues/5758)) - @yangchiu @DamiaSan - [FEATURE] Add a global setting for enabling and disabling SPDK feature ([5778](https://github.com/longhorn/longhorn/issues/5778)) - @yangchiu @derekbit - [FEATURE] Identify and manage orphaned lvols and raid bdevs if the associated `Volume` resources are not existing ([5827](https://github.com/longhorn/longhorn/issues/5827)) - @yangchiu @derekbit - [FEATURE] Longhorn UI for SPDK feature ([5846](https://github.com/longhorn/longhorn/issues/5846)) - @smallteeths @chriscchien - [FEATURE] UI modification to work with new AD mechanism (Longhorn UI -> Longhorn API) ([6004](https://github.com/longhorn/longhorn/issues/6004)) - @yangchiu @smallteeths - [FEATURE] Replica offline rebuild over SPDK - data engine ([6067](https://github.com/longhorn/longhorn/issues/6067)) - @shuo-wu - [FEATURE] Support automatic offline replica rebuilding of volumes using SPDK data engine ([6071](https://github.com/longhorn/longhorn/issues/6071)) - @yangchiu @derekbit ## Improvement - [IMPROVEMENT] Do not count the failure replica reuse failure caused by the disconnection ([1923](https://github.com/longhorn/longhorn/issues/1923)) - @yangchiu @mantissahz - [IMPROVEMENT] Consider changing the over provisioning default/recommendation to 100% percentage (no over provisioning) ([2694](https://github.com/longhorn/longhorn/issues/2694)) - @c3y1huang @chriscchien - [BUG] StorageClass of pv and pvc of a recovered pv should not always be default. ([3506](https://github.com/longhorn/longhorn/issues/3506)) - @ChanYiLin @smallteeths @roger-ryao - [IMPROVEMENT] Auto-attach volume for K8s CSI snapshot ([3726](https://github.com/longhorn/longhorn/issues/3726)) - @weizhe0422 @PhanLe1010 - [IMPROVEMENT] Change Longhorn API to create/delete snapshot CRs instead of calling engine CLI ([3995](https://github.com/longhorn/longhorn/issues/3995)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Add support for crypto parameters for RWX volumes ([4829](https://github.com/longhorn/longhorn/issues/4829)) - @mantissahz @roger-ryao - [IMPROVEMENT] Remove the global setting `mkfs-ext4-parameters` ([4914](https://github.com/longhorn/longhorn/issues/4914)) - @ejweber @roger-ryao - [IMPROVEMENT] Move all snapshot related settings at one place. ([4930](https://github.com/longhorn/longhorn/issues/4930)) - @smallteeths @roger-ryao - [IMPROVEMENT] Remove system managed component image settings ([5028](https://github.com/longhorn/longhorn/issues/5028)) - @mantissahz @chriscchien - [IMPROVEMENT] Set default `engine-replica-timeout` value for engine controller start command ([5031](https://github.com/longhorn/longhorn/issues/5031)) - @derekbit @chriscchien - [IMPROVEMENT] Support bundle collects dmesg, syslog and related information of longhorn nodes ([5073](https://github.com/longhorn/longhorn/issues/5073)) - @weizhe0422 @roger-ryao - [IMPROVEMENT] Collect volume, system, feature info for metrics for better usage awareness ([5235](https://github.com/longhorn/longhorn/issues/5235)) - @c3y1huang @chriscchien @roger-ryao - [IMPROVEMENT] Update uninstallation info to include the 'Deleting Confirmation Flag' in chart ([5250](https://github.com/longhorn/longhorn/issues/5250)) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Disable Revision Counter for Strict-Local dataLocality ([5257](https://github.com/longhorn/longhorn/issues/5257)) - @derekbit @roger-ryao - [IMPROVEMENT] Fix Guaranteed Engine Manager CPU recommendation formula in UI ([5338](https://github.com/longhorn/longhorn/issues/5338)) - @c3y1huang @smallteeths @roger-ryao - [IMPROVEMENT] Update PSP validation in the Longhorn upstream chart ([5339](https://github.com/longhorn/longhorn/issues/5339)) - @yangchiu @PhanLe1010 - [IMPROVEMENT] Update ganesha nfs to 4.2.3 ([5356](https://github.com/longhorn/longhorn/issues/5356)) - @derekbit @roger-ryao - [IMPROVEMENT] Set write-cache of longhorn block device to off explicitly ([5382](https://github.com/longhorn/longhorn/issues/5382)) - @derekbit @chriscchien - [IMPROVEMENT] Clean up unused backupstore mountpoint ([5391](https://github.com/longhorn/longhorn/issues/5391)) - @derekbit @chriscchien - [DOC] Update Kubernetes version info to have consistent description from the longhorn documentation in chart ([5399](https://github.com/longhorn/longhorn/issues/5399)) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Fix BackingImage uploading/downloading flow to prevent client timeout ([5443](https://github.com/longhorn/longhorn/issues/5443)) - @ChanYiLin @chriscchien - [IMPROVEMENT] Assign the pods to the same node where the strict-local volume is present ([5448](https://github.com/longhorn/longhorn/issues/5448)) - @c3y1huang @chriscchien - [IMPROVEMENT] Have explicitly message when trying to attach a volume which it's engine and replica were on deleted node ([5545](https://github.com/longhorn/longhorn/issues/5545)) - @ChanYiLin @chriscchien - [IMPROVEMENT] Create a new setting so that Longhorn removes PDB for instance-manager-r that doesn't have any running instance inside it ([5549](https://github.com/longhorn/longhorn/issues/5549)) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Merge conversion/admission webhook and recovery backend services into longhorn-manager ([5590](https://github.com/longhorn/longhorn/issues/5590)) - @ChanYiLin @chriscchien - [IMPROVEMENT][UI] Recurring jobs create new snapshots while being not able to clean up old one ([5610](https://github.com/longhorn/longhorn/issues/5610)) - @mantissahz @smallteeths @roger-ryao - [IMPROVEMENT] Only activate replica if it doesn't have deletion timestamp during volume engine upgrade ([5632](https://github.com/longhorn/longhorn/issues/5632)) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Clean up backup target if the backup target setting is unset ([5655](https://github.com/longhorn/longhorn/issues/5655)) - @yangchiu @ChanYiLin - [IMPROVEMENT] Bump CSI sidecar components' version ([5672](https://github.com/longhorn/longhorn/issues/5672)) - @yangchiu @ejweber - [IMPROVEMENT] Configure log level of Longhorn components ([5888](https://github.com/longhorn/longhorn/issues/5888)) - @ChanYiLin @weizhe0422 - [IMPROVEMENT] Remove development toolchain from Longhorn images ([6022](https://github.com/longhorn/longhorn/issues/6022)) - @ChanYiLin @derekbit - [IMPROVEMENT] Reduce replica process's number of allocated ports ([6079](https://github.com/longhorn/longhorn/issues/6079)) - @ChanYiLin @derekbit - [IMPROVEMENT] UI supports automatic replica rebuilding for SPDK volumes ([6107](https://github.com/longhorn/longhorn/issues/6107)) - @smallteeths @roger-ryao - [IMPROVEMENT] Minor UX changes for Longhorn SPDK ([6126](https://github.com/longhorn/longhorn/issues/6126)) - @derekbit @roger-ryao - [IMPROVEMENT] Instance manager spdk_tgt resilience due to spdk_tgt crash ([6155](https://github.com/longhorn/longhorn/issues/6155)) - @yangchiu @derekbit - [IMPROVEMENT] Determine number of replica/engine port count in longhorn-manager (control plane) instead ([6163](https://github.com/longhorn/longhorn/issues/6163)) - @derekbit @chriscchien - [IMPROVEMENT] SPDK client should functions after encountering decoding error ([6191](https://github.com/longhorn/longhorn/issues/6191)) - @yangchiu @shuo-wu ## Performance - [REFACTORING] Evaluate the impact of removing the client side compression for backup blocks ([1409](https://github.com/longhorn/longhorn/issues/1409)) - @derekbit ## Resilience - [BUG] If backing image downloading fails on one node, it doesn't try on other nodes. ([3746](https://github.com/longhorn/longhorn/issues/3746)) - @ChanYiLin - [BUG] Replica rebuilding caused by rke2/kubelet restart ([5340](https://github.com/longhorn/longhorn/issues/5340)) - @derekbit @chriscchien - [BUG] Volume restoration will never complete if attached node is down ([5464](https://github.com/longhorn/longhorn/issues/5464)) - @derekbit @weizhe0422 @chriscchien - [BUG] Node disconnection test failed ([5476](https://github.com/longhorn/longhorn/issues/5476)) - @yangchiu @derekbit - [BUG] Physical node down test failed ([5477](https://github.com/longhorn/longhorn/issues/5477)) - @derekbit @chriscchien - [BUG] Backing image with sync failure ([5481](https://github.com/longhorn/longhorn/issues/5481)) - @ChanYiLin @roger-ryao - [BUG] share-manager pod failed to restart after kubelet restart ([5507](https://github.com/longhorn/longhorn/issues/5507)) - @yangchiu @derekbit - [BUG] Directly mark replica as failed if the node is deleted ([5542](https://github.com/longhorn/longhorn/issues/5542)) - @weizhe0422 @roger-ryao - [BUG] RWX volume is stuck at detaching when the attached node is down ([5558](https://github.com/longhorn/longhorn/issues/5558)) - @derekbit @roger-ryao - [BUG] Unable to export RAID1 bdev in degraded state ([5650](https://github.com/longhorn/longhorn/issues/5650)) - @chriscchien @DamiaSan - [BUG] Backup monitor gets stuck in an infinite loop if backup isn't found ([5662](https://github.com/longhorn/longhorn/issues/5662)) - @derekbit @chriscchien - [BUG] Resources such as replicas are somehow not mutated when network is unstable ([5762](https://github.com/longhorn/longhorn/issues/5762)) - @derekbit @roger-ryao - [BUG] filesystem corrupted after delete instance-manager-r for a locality best-effort volume ([5801](https://github.com/longhorn/longhorn/issues/5801)) - @yangchiu @ChanYiLin @mantissahz ## Stability - [BUG] nfs backup broken - NFS server: mkdir - file exists ([4626](https://github.com/longhorn/longhorn/issues/4626)) - @yangchiu @derekbit - [BUG] Memory leak in CSI plugin caused by stuck umount processes if the RWX volume is already gone ([5296](https://github.com/longhorn/longhorn/issues/5296)) - @derekbit @roger-ryao ## Bugs - [BUG] 'Upgrade Engine' still shows up in a specific situation when engine already upgraded ([3063](https://github.com/longhorn/longhorn/issues/3063)) - @weizhe0422 @PhanLe1010 @smallteeths - [BUG] DR volume even after activation remains in standby mode if there are one or more failed replicas. ([3069](https://github.com/longhorn/longhorn/issues/3069)) - @yangchiu @mantissahz - [BUG] volume not able to attach with raw type backing image ([3437](https://github.com/longhorn/longhorn/issues/3437)) - @yangchiu @ChanYiLin - [BUG] Delete a uploading backing image, the corresponding LH temp file is not deleted ([3682](https://github.com/longhorn/longhorn/issues/3682)) - @ChanYiLin @chriscchien - [BUG] Cloned PVC from detached volume will stuck at not ready for workload ([3692](https://github.com/longhorn/longhorn/issues/3692)) - @PhanLe1010 @chriscchien - [BUG] Block device volume failed to unmount when it is detached unexpectedly ([3778](https://github.com/longhorn/longhorn/issues/3778)) - @PhanLe1010 @chriscchien - [BUG] After migration of Longhorn from Rancher old UI to dashboard, the csi-plugin doesn't update ([4519](https://github.com/longhorn/longhorn/issues/4519)) - @mantissahz @roger-ryao - [BUG] Volumes Stuck in Attach/Detach Loop when running on OpenShift/OKD ([4988](https://github.com/longhorn/longhorn/issues/4988)) - @ChanYiLin - [BUG] Longhorn 1.3.2 fails to backup & restore volumes behind Internet proxy ([5054](https://github.com/longhorn/longhorn/issues/5054)) - @mantissahz @chriscchien - [BUG] Instance manager pod does not respect of node taint? ([5161](https://github.com/longhorn/longhorn/issues/5161)) - @ejweber - [BUG] RWX doesn't work with release 1.4.0 due to end grace update error from recovery backend ([5183](https://github.com/longhorn/longhorn/issues/5183)) - @derekbit @chriscchien - [BUG] Incorrect indentation of charts/questions.yaml ([5196](https://github.com/longhorn/longhorn/issues/5196)) - @mantissahz @roger-ryao - [BUG] Updating option "Allow snapshots removal during trim" for old volumes failed ([5218](https://github.com/longhorn/longhorn/issues/5218)) - @shuo-wu @roger-ryao - [BUG] Since 1.4.0 RWX volume failing regularly ([5224](https://github.com/longhorn/longhorn/issues/5224)) - @derekbit - [BUG] Can not create backup in engine image not fully deployed cluster ([5248](https://github.com/longhorn/longhorn/issues/5248)) - @ChanYiLin @roger-ryao - [BUG] Incorrect router retry mechanism ([5259](https://github.com/longhorn/longhorn/issues/5259)) - @mantissahz @chriscchien - [BUG] System Backup is stuck at Uploading if there are PVs not provisioned by CSI driver ([5286](https://github.com/longhorn/longhorn/issues/5286)) - @c3y1huang @chriscchien - [BUG] Sync up with backup target during DR volume activation ([5292](https://github.com/longhorn/longhorn/issues/5292)) - @yangchiu @weizhe0422 - [BUG] environment_check.sh does not handle different kernel versions in cluster correctly ([5304](https://github.com/longhorn/longhorn/issues/5304)) - @achims311 @roger-ryao - [BUG] instance-manager-r high memory consumption ([5312](https://github.com/longhorn/longhorn/issues/5312)) - @derekbit @roger-ryao - [BUG] Unable to upgrade longhorn from v1.3.2 to master-head ([5368](https://github.com/longhorn/longhorn/issues/5368)) - @yangchiu @derekbit - [BUG] Modify engineManagerCPURequest and replicaManagerCPURequest won't raise resource request in instance-manager-e pod ([5419](https://github.com/longhorn/longhorn/issues/5419)) - @c3y1huang - [BUG] Error message not consistent between create/update recurring job when retain number greater than 50 ([5434](https://github.com/longhorn/longhorn/issues/5434)) - @c3y1huang @chriscchien - [BUG] Do not copy Host header to API requests forwarded to Longhorn Manager ([5438](https://github.com/longhorn/longhorn/issues/5438)) - @yangchiu @smallteeths - [BUG] RWX Volume attachment is getting Failed ([5456](https://github.com/longhorn/longhorn/issues/5456)) - @derekbit - [BUG] test case test_backup_lock_deletion_during_restoration failed ([5458](https://github.com/longhorn/longhorn/issues/5458)) - @yangchiu @derekbit - [BUG] Unable to create support bundle agent pod in air-gap environment ([5467](https://github.com/longhorn/longhorn/issues/5467)) - @yangchiu @c3y1huang - [BUG] Example of data migration doesn't work for hidden/./dot-files) ([5484](https://github.com/longhorn/longhorn/issues/5484)) - @hedefalk @shuo-wu @chriscchien - [BUG] Upgrade engine --> spec.restoreVolumeRecurringJob and spec.snapshotDataIntegrity Unsupported value ([5485](https://github.com/longhorn/longhorn/issues/5485)) - @yangchiu @derekbit - [BUG] test case test_dr_volume_with_backup_block_deletion failed ([5489](https://github.com/longhorn/longhorn/issues/5489)) - @yangchiu @derekbit - [BUG] Bulk backup deletion cause restoring volume to finish with attached state. ([5506](https://github.com/longhorn/longhorn/issues/5506)) - @ChanYiLin @roger-ryao - [BUG] volume expansion starts for no reason, gets stuck on current size > expected size ([5513](https://github.com/longhorn/longhorn/issues/5513)) - @mantissahz @roger-ryao - [BUG] RWX volume attachment failed if tried more enough times ([5537](https://github.com/longhorn/longhorn/issues/5537)) - @yangchiu @derekbit - [BUG] instance-manager-e emits `Wait for process pvc-xxxx to shutdown` constantly ([5575](https://github.com/longhorn/longhorn/issues/5575)) - @derekbit @roger-ryao - [BUG] Support bundle kit should respect node selector & taint toleration ([5614](https://github.com/longhorn/longhorn/issues/5614)) - @yangchiu @c3y1huang - [BUG] Value overlapped in page Instance Manager Image ([5622](https://github.com/longhorn/longhorn/issues/5622)) - @smallteeths @chriscchien - [BUG] Updated Rocky 9 (and others) can't attach due to SELinux ([5627](https://github.com/longhorn/longhorn/issues/5627)) - @yangchiu @ejweber - [BUG] Fix misleading error messages when creating a mount point for a backup store ([5630](https://github.com/longhorn/longhorn/issues/5630)) - @derekbit - [BUG] Instance manager PDB created with wrong selector thus blocking the draining of the wrongly selected node forever ([5680](https://github.com/longhorn/longhorn/issues/5680)) - @PhanLe1010 @chriscchien - [BUG] During volume live engine upgrade, if the replica pod is killed, the volume is stuck in upgrading forever ([5684](https://github.com/longhorn/longhorn/issues/5684)) - @yangchiu @PhanLe1010 - [BUG] Instance manager PDBs cannot be removed if the longhorn-manager pod on its spec node is not available ([5688](https://github.com/longhorn/longhorn/issues/5688)) - @PhanLe1010 @roger-ryao - [BUG] Rebuild rebuilding is possibly issued to a wrong replica ([5709](https://github.com/longhorn/longhorn/issues/5709)) - @ejweber @roger-ryao - [BUG] Observing repilca on new IM-r before upgrading of volume ([5729](https://github.com/longhorn/longhorn/issues/5729)) - @c3y1huang - [BUG] longhorn upgrade is not upgrading engineimage ([5740](https://github.com/longhorn/longhorn/issues/5740)) - @shuo-wu @chriscchien - [BUG] `test_replica_auto_balance_when_replica_on_unschedulable_node` Error in creating volume with nodeSelector and dataLocality parameters ([5745](https://github.com/longhorn/longhorn/issues/5745)) - @c3y1huang @roger-ryao - [BUG] Unable to backup volume after NFS server IP change ([5856](https://github.com/longhorn/longhorn/issues/5856)) - @derekbit @roger-ryao - [BUG] Prevent Longhorn uninstallation from getting stuck due to backups in error ([5868](https://github.com/longhorn/longhorn/issues/5868)) - @ChanYiLin @mantissahz - [BUG] Unable to create support bundle if the previous one stayed in ReadyForDownload phase ([5882](https://github.com/longhorn/longhorn/issues/5882)) - @c3y1huang @roger-ryao - [BUG] share-manager for a given pvc keep restarting (other pvc are working fine) ([5954](https://github.com/longhorn/longhorn/issues/5954)) - @yangchiu @derekbit - [BUG] Replica auto-rebalance doesn't respect node selector ([5971](https://github.com/longhorn/longhorn/issues/5971)) - @c3y1huang @roger-ryao - [BUG] Volume detached automatically after upgrade Longhorn ([5983](https://github.com/longhorn/longhorn/issues/5983)) - @yangchiu @PhanLe1010 - [BUG] Extra snapshot generated when clone from a detached volume ([5986](https://github.com/longhorn/longhorn/issues/5986)) - @weizhe0422 @ejweber - [BUG] User created snapshot deleted after node drain and uncordon ([5992](https://github.com/longhorn/longhorn/issues/5992)) - @yangchiu @mantissahz - [BUG] Webhook PDBs are not removed after upgrading to master-head ([6026](https://github.com/longhorn/longhorn/issues/6026)) - @weizhe0422 @PhanLe1010 - [BUG] In some specific situation, system backup auto deleted when creating another one ([6045](https://github.com/longhorn/longhorn/issues/6045)) - @c3y1huang @chriscchien - [BUG] Backing Image deletion stuck if it's deleted during uploading process and bids is ready-for-transfer state ([6086](https://github.com/longhorn/longhorn/issues/6086)) - @WebberHuang1118 @chriscchien - [BUG] A backup target backed by a Samba server is not recognized ([6100](https://github.com/longhorn/longhorn/issues/6100)) - @derekbit @weizhe0422 - [BUG] Backing image manager fails when SELinux is enabled ([6108](https://github.com/longhorn/longhorn/issues/6108)) - @ejweber @chriscchien - [BUG] Force delete volume make SPDK disk unschedule ([6110](https://github.com/longhorn/longhorn/issues/6110)) - @derekbit - [BUG] share-manager terminated during Longhorn upgrading causes rwx volume not working ([6120](https://github.com/longhorn/longhorn/issues/6120)) - @yangchiu @derekbit - [BUG] SPDK Volume snapshotList API Error ([6123](https://github.com/longhorn/longhorn/issues/6123)) - @derekbit @chriscchien - [BUG] test_recurring_jobs_allow_detached_volume failed ([6124](https://github.com/longhorn/longhorn/issues/6124)) - @ChanYiLin @roger-ryao - [BUG] Cron job triggered replica rebuilding keeps repeating itself after corrupting snapshot data ([6129](https://github.com/longhorn/longhorn/issues/6129)) - @yangchiu @mantissahz - [BUG] test_dr_volume_with_restore_command_error failed ([6130](https://github.com/longhorn/longhorn/issues/6130)) - @mantissahz @roger-ryao - [BUG] RWX volume remains attached after workload deleted if it's upgraded from v1.4.2 ([6139](https://github.com/longhorn/longhorn/issues/6139)) - @PhanLe1010 @chriscchien - [BUG] timestamp or checksum not matched in test_snapshot_hash_detect_corruption test case ([6145](https://github.com/longhorn/longhorn/issues/6145)) - @yangchiu @derekbit - [BUG] When a v2 volume is attached in maintenance mode, removing a replica will lead to volume stuck in attaching-detaching loop ([6166](https://github.com/longhorn/longhorn/issues/6166)) - @derekbit @chriscchien - [BUG] Misleading offline rebuilding hint if offline rebuilding is not enabled ([6169](https://github.com/longhorn/longhorn/issues/6169)) - @smallteeths @roger-ryao - [BUG] Longhorn doesn't remove the system backups crd on uninstallation ([6185](https://github.com/longhorn/longhorn/issues/6185)) - @c3y1huang @khushboo-rancher - [BUG] Volume attachment related error logs in uninstaller pod ([6197](https://github.com/longhorn/longhorn/issues/6197)) - @yangchiu @PhanLe1010 - [BUG] Test case test_ha_backup_deletion_recovery failed in rhel or rockylinux arm64 environment ([6213](https://github.com/longhorn/longhorn/issues/6213)) - @yangchiu @ChanYiLin @mantissahz - [BUG] migration test cases could fail due to unexpected volume controllers and replicas status ([6215](https://github.com/longhorn/longhorn/issues/6215)) - @yangchiu @PhanLe1010 - [BUG] Engine continues to attempt to rebuild replica while detaching ([6217](https://github.com/longhorn/longhorn/issues/6217)) - @yangchiu @ejweber ## Misc - [TASK] Remove deprecated volume spec recurringJobs and storageClass recurringJobs field ([2865](https://github.com/longhorn/longhorn/issues/2865)) - @c3y1huang @chriscchien - [TASK] Remove deprecated fields after CRD API version bump ([3289](https://github.com/longhorn/longhorn/issues/3289)) - @c3y1huang @roger-ryao - [TASK] Replace jobq lib with an alternative way for listing remote backup volumes and info ([4176](https://github.com/longhorn/longhorn/issues/4176)) - @ChanYiLin @chriscchien - [DOC] Update the Longhorn document in Uninstalling Longhorn using kubectl ([4841](https://github.com/longhorn/longhorn/issues/4841)) - @roger-ryao - [TASK] Remove a deprecated feature `disable-replica-rebuild` from longhorn-manager ([4997](https://github.com/longhorn/longhorn/issues/4997)) - @ejweber @chriscchien - [TASK] Update the distro matrix supports on Longhorn docs for 1.5 ([5177](https://github.com/longhorn/longhorn/issues/5177)) - @yangchiu - [TASK] Clarify if any upcoming K8s API deprecation/removal will impact Longhorn 1.4 ([5180](https://github.com/longhorn/longhorn/issues/5180)) - @PhanLe1010 - [TASK] Revert affinity for Longhorn user deployed components ([5191](https://github.com/longhorn/longhorn/issues/5191)) - @weizhe0422 @ejweber - [TASK] Add GitHub action for CI to lib repos for supporting dependency bot ([5239](https://github.com/longhorn/longhorn/issues/5239)) - - [DOC] Update the readme of longhorn-spdk-engine about using new Longhorn (RAID1) bdev ([5256](https://github.com/longhorn/longhorn/issues/5256)) - @DamiaSan - [TASK][UI] add new recurring job tasks ([5272](https://github.com/longhorn/longhorn/issues/5272)) - @smallteeths @chriscchien - [DOC] Update the node maintenance doc to cover upgrade prerequisites for Rancher ([5278](https://github.com/longhorn/longhorn/issues/5278)) - @PhanLe1010 - [TASK] Run build-engine-test-images automatically when having incompatible engine on master ([5400](https://github.com/longhorn/longhorn/issues/5400)) - @yangchiu - [TASK] Update k8s.gcr.io to registry.k8s.io in repos ([5432](https://github.com/longhorn/longhorn/issues/5432)) - @yangchiu - [TASK][UI] add new recurring job task - filesystem trim ([5529](https://github.com/longhorn/longhorn/issues/5529)) - @smallteeths @chriscchien - doc: update prerequisites in chart readme to make it consistent with documentation v1.3.x ([5531](https://github.com/longhorn/longhorn/pull/5531)) - @ChanYiLin - [FEATURE] Remove deprecated `allow-node-drain-with-last-healthy-replica` ([5620](https://github.com/longhorn/longhorn/issues/5620)) - @weizhe0422 @PhanLe1010 - [FEATURE] Set recurring jobs to PVCs ([5791](https://github.com/longhorn/longhorn/issues/5791)) - @yangchiu @c3y1huang - [TASK] Automatically update crds.yaml in longhorn repo from longhorn-manager repo ([5854](https://github.com/longhorn/longhorn/issues/5854)) - @yangchiu - [IMPROVEMENT] Remove privilege requirement from lifecycle jobs ([5862](https://github.com/longhorn/longhorn/issues/5862)) - @mantissahz @chriscchien - [TASK][UI] support new aio typed instance managers ([5876](https://github.com/longhorn/longhorn/issues/5876)) - @smallteeths @chriscchien - [TASK] Remove `Guaranteed Engine Manager CPU`, `Guaranteed Replica Manager CPU`, and `Guaranteed Engine CPU` settings. ([5917](https://github.com/longhorn/longhorn/issues/5917)) - @c3y1huang @roger-ryao - [TASK][UI] Support volume backup policy ([6028](https://github.com/longhorn/longhorn/issues/6028)) - @smallteeths @chriscchien - [TASK] Reduce BackupConcurrentLimit and RestoreConcurrentLimit default values ([6135](https://github.com/longhorn/longhorn/issues/6135)) - @derekbit @chriscchien ## Contributors - @ChanYiLin - @DamiaSan - @PhanLe1010 - @WebberHuang1118 - @achims311 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @hedefalk - @innobead - @khushboo-rancher - @mantissahz - @roger-ryao - @shuo-wu - @smallteeths - @weizhe0422 - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.1.md ================================================ ## Release Note ### **v1.5.1 released!** 🎆 Longhorn v1.5.1 is the latest version of Longhorn 1.5. This release introduces bug fixes as described below about 1.5.0 upgrade issues, stability, troubleshooting and so on. Please try it and feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing v1.5.1.** Longhorn supports 3 installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.5.1/deploy/install/). ## Upgrade > **Please read the [important notes](https://longhorn.io/docs/1.5.1/deploy/important-notes/) first and ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.5.1 from v1.4.x/v1.5.0, which are only supported source versions.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.5.1/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Improvement - [IMPROVEMENT] Implement/fix the unit tests of Volume Attachment and volume controller ([6005](https://github.com/longhorn/longhorn/issues/6005)) - @PhanLe1010 - [QUESTION] Repetitive warnings and errors in a new longhorn setup ([6257](https://github.com/longhorn/longhorn/issues/6257)) - @derekbit @c3y1huang @roger-ryao ## Resilience - [BUG] 1.5.0 Upgrade: Longhorn conversion webhook server fails ([6259](https://github.com/longhorn/longhorn/issues/6259)) - @derekbit @roger-ryao - [BUG] Race leaves snapshot CRs that cannot be deleted ([6298](https://github.com/longhorn/longhorn/issues/6298)) - @yangchiu @PhanLe1010 @ejweber ## Bugs - [BUG] Engine continues to attempt to rebuild replica while detaching ([6217](https://github.com/longhorn/longhorn/issues/6217)) - @yangchiu @ejweber - [BUG] Upgrade to 1.5.0 failed: validator.longhorn.io denied the request if having orphan resources ([6246](https://github.com/longhorn/longhorn/issues/6246)) - @derekbit @roger-ryao - [BUG] Unable to receive support bundle from UI when it's large (400MB+) ([6256](https://github.com/longhorn/longhorn/issues/6256)) - @c3y1huang @chriscchien - [BUG] Longhorn Manager Pods CrashLoop after upgrade from 1.4.0 to 1.5.0 while backing up volumes ([6264](https://github.com/longhorn/longhorn/issues/6264)) - @ChanYiLin @roger-ryao - [BUG] Can not delete type=`bi` VolumeSnapshot if related backing image not exist ([6266](https://github.com/longhorn/longhorn/issues/6266)) - @ChanYiLin @chriscchien - [BUG] 1.5.0: AttachVolume.Attach failed for volume, the volume is currently attached to a different node ([6287](https://github.com/longhorn/longhorn/issues/6287)) - @yangchiu @derekbit - [BUG] test case test_setting_priority_class failed in master and v1.5.x ([6319](https://github.com/longhorn/longhorn/issues/6319)) - @derekbit @chriscchien - [BUG] Unused webhook and recovery backend deployment left in helm chart ([6252](https://github.com/longhorn/longhorn/issues/6252)) - @ChanYiLin @chriscchien ## Misc - [DOC] v1.5.0 additional outgoing firewall ports need to be opened 9501 9502 9503 ([6317](https://github.com/longhorn/longhorn/issues/6317)) - @ChanYiLin @chriscchien ## Contributors - @ChanYiLin - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @roger-ryao - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.2.md ================================================ ## Release Note ### **v1.5.2 released!** 🎆 This release introduces bug fixes and improvements, with the main focus on stability. Please try it and provide feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing v1.5.2.** Longhorn supports three installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.5.2/deploy/install/). ## Upgrade > **Please read the [important notes](https://longhorn.io/docs/1.5.2/deploy/important-notes/) first and ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.5.2 from v1.4.x/v1.5.x, which are only supported source versions.** Follow the upgrade instructions here. [Here](https://longhorn.io/docs/1.5.2/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Resolved Issues ### Enhancement - [FEATURE] Add disk status prometheus metrics [6858](https://github.com/longhorn/longhorn/issues/6858) - @c3y1huang @chriscchien ### Improvement - [IMPROVEMENT] High memory consumption of longhorn-manager pods since Longhorn v1.5 [6936](https://github.com/longhorn/longhorn/issues/6936) - @derekbit - [IMPROVEMENT] Old kernel such as 3.10.0 set provisioning_mode to wrong value (writesame_16, disabled, full, ...) but not the correct value (unmap) so the trim feature doesn't work [6854](https://github.com/longhorn/longhorn/issues/6854) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Improve log level for resource update failure able to reconcile again [6843](https://github.com/longhorn/longhorn/issues/6843) - @PhanLe1010 @nitendra-suse - [IMPROVEMENT] Don't log about inability to change settings that didn't change. [6812](https://github.com/longhorn/longhorn/issues/6812) - @james-munson @roger-ryao - [IMPROVEMENT] Use nvme-cli in instance-manager pod instead [6798](https://github.com/longhorn/longhorn/issues/6798) - @derekbit - [IMPROVEMENT] Prevent unexpected engine creation [6682](https://github.com/longhorn/longhorn/issues/6682) - @PhanLe1010 @ejweber @roger-ryao - [IMPROVEMENT] Support both NFS `hard` and `soft` with custom `timeo` and `retrans` options for RWX volumes [6655](https://github.com/longhorn/longhorn/issues/6655) - @derekbit @roger-ryao - [IMPROVEMENT] Prevent Volume Provision if Related Backing Image Stuck in Ready-For-Trasfer State [6615](https://github.com/longhorn/longhorn/issues/6615) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Support custom options for network filesystems for backup [6608](https://github.com/longhorn/longhorn/issues/6608) - @james-munson - [IMPROVEMENT] Remove dummy services of each CSI sidecar if not required [6581](https://github.com/longhorn/longhorn/issues/6581) - @ejweber @roger-ryao - [IMPROVEMENT] Include /var/log/messages during the support-bundle syslog collection [6544](https://github.com/longhorn/longhorn/issues/6544) - @c3y1huang @roger-ryao - [IMPROVEMENT] Provide more information for volume scheduling failure [6461](https://github.com/longhorn/longhorn/issues/6461) - @smallteeths @chriscchien - [IMPROVEMENT] Remove or Change Helm pre-upgrade hook to support ArgoCD [6415](https://github.com/longhorn/longhorn/issues/6415) - @mantissahz - [IMPROVEMENT] Improve upgrade path and make it more solid [6294](https://github.com/longhorn/longhorn/issues/6294) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] UI Volume detail page still shows `Block Device` when `spec.disableFrontend` is true [6167](https://github.com/longhorn/longhorn/issues/6167) - @smallteeths @chriscchien - [IMPROVEMENT] Fix scheduling flooding logs [6019](https://github.com/longhorn/longhorn/issues/6019) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Add reserve storage percentage of nodes setting in helm chart [5958](https://github.com/longhorn/longhorn/issues/5958) - @mantissahz @roger-ryao - [IMPROVEMENT] Longhorn-engine processes should refuse to serve requests not intended for them [5845](https://github.com/longhorn/longhorn/issues/5845) - @ejweber @chriscchien - [IMPROVEMENT] Unify logs with extra static info like module/method/function/line [5509](https://github.com/longhorn/longhorn/issues/5509) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Add pvc name to longhorn_volume metrics [5297](https://github.com/longhorn/longhorn/issues/5297) - @c3y1huang @nitendra-suse - [IMPROVEMENT] Avoid the accident deletion of longhorn settings [4984](https://github.com/longhorn/longhorn/issues/4984) - @ejweber @roger-ryao - [IMPROVEMENT] Remove Longhorn engine path mismatch log [3786](https://github.com/longhorn/longhorn/issues/3786) - @c3y1huang @roger-ryao ### Performance - [IMPROVEMENT] Optimize the resource cache to prevent high memory usage in longhorn-manager [6954](https://github.com/longhorn/longhorn/issues/6954) - @derekbit @nitendra-suse - [BUG] Longhorn manager pods in 1.5.1 consuming 20GB+ RAM and 3-4 vCPUs [6866](https://github.com/longhorn/longhorn/issues/6866) - @derekbit @shuo-wu - [BUG] Longhorn Instance Manager Memory leak [6481](https://github.com/longhorn/longhorn/issues/6481) - @james-munson @chriscchien ### Stability - [BUG] DR volume failed when synchronizing the incremental backup [6750](https://github.com/longhorn/longhorn/issues/6750) - @mantissahz @chriscchien - [BUG] After crashed engine process, volume stuck in `Unknown` state [6699](https://github.com/longhorn/longhorn/issues/6699) - @ChanYiLin @nitendra-suse - [BUG] Two active engine when volume migrating [6642](https://github.com/longhorn/longhorn/issues/6642) - @PhanLe1010 @chriscchien - [BUG] Somehow the Rebuilding field inside volume.meta is set to true when one replica only, causing the volume into attaching/detaching loop [6626](https://github.com/longhorn/longhorn/issues/6626) - @c3y1huang @nitendra-suse ### Resilience - [BUG] RWX workload gets stuck in ContainerCreating after cluster restart [6924](https://github.com/longhorn/longhorn/issues/6924) - @yangchiu @derekbit - [BUG] Volumes failing to mount because of engine upgradedReplicaAddressMap reference [6762](https://github.com/longhorn/longhorn/issues/6762) - @PhanLe1010 @chriscchien - [BUG] Set a invalid backup target when backup in progress will cause backup never finish [6491](https://github.com/longhorn/longhorn/issues/6491) - @ChanYiLin @chriscchien - [BUG] Share manager pod will stay in IO error when the volume becomes read only [5961](https://github.com/longhorn/longhorn/issues/5961) - @ChanYiLin @roger-ryao ### Bug - [BUG] duplicate MIME type "text/html" in `/var/config/nginx/nginx.conf` [7002](https://github.com/longhorn/longhorn/issues/7002) - @votdev - [BUG] invalid memory address or nil pointer dereference in BackupVolumeController [6998](https://github.com/longhorn/longhorn/issues/6998) - @mantissahz @roger-ryao - [BUG] Unable to upgrade longhorn from v1.4.x to v1.5.x: longhorn-manager CrashLoopBackOff [6987](https://github.com/longhorn/longhorn/issues/6987) - @mantissahz @chriscchien - [BUG] Longhorn storage network is incompatible with Multus version above v4.0.0 [6953](https://github.com/longhorn/longhorn/issues/6953) - @c3y1huang - [BUG] longhorn manager isn't annotated with iam.amazonaws.com/role [6947](https://github.com/longhorn/longhorn/issues/6947) - @mantissahz - [BUG] Unable to add a block-type disk with a new name [6849](https://github.com/longhorn/longhorn/issues/6849) - @derekbit @chriscchien - [BUG] IO error occurs when detaching RWX volume [6829](https://github.com/longhorn/longhorn/issues/6829) - @derekbit @chriscchien - [BUG] cifs backup mount paths with dollar sign are not allowed [6660](https://github.com/longhorn/longhorn/issues/6660) - @derekbit @roger-ryao - [BUG] Orphan snapshot attachment tickets prevent volume from detaching [6652](https://github.com/longhorn/longhorn/issues/6652) - @ejweber - [BUG] High CPU usage on one node. [6578](https://github.com/longhorn/longhorn/issues/6578) - @derekbit @chriscchien - [BUG] The instance manager with state unknown will be cleaned up in the split-brain case [6479](https://github.com/longhorn/longhorn/issues/6479) - @shuo-wu - [BUG] Removed IM CPU request settings still exists and new IM CPU request missed from chart settings [6465](https://github.com/longhorn/longhorn/issues/6465) - @c3y1huang @chriscchien - [BUG] PV using v2 engine cannot attach [6441](https://github.com/longhorn/longhorn/issues/6441) - @derekbit @chriscchien @nitendra-suse - [BUG] SettingNameSnapshotDataIntegrityCronJob should be sent as boolean value [6410](https://github.com/longhorn/longhorn/issues/6410) - @c3y1huang @roger-ryao - [BUG] Fix errors in questions.yaml [6392](https://github.com/longhorn/longhorn/issues/6392) - @james-munson @chriscchien - [BUG] Webhook is never called for BackingImageManager [6328](https://github.com/longhorn/longhorn/issues/6328) - @ejweber @chriscchien - [BUG] Longhorn Read-Only setting can be modified [5989](https://github.com/longhorn/longhorn/issues/5989) - @mantissahz - [BUG] Environment Check Script Fails To Perform All Checks [5653](https://github.com/longhorn/longhorn/issues/5653) - @PhanLe1010 @roger-ryao - [BUG] Can't delete volumesnapshot if backup target not set [4979](https://github.com/longhorn/longhorn/issues/4979) - @ejweber @chriscchien - [BUG] Backup Job returns "Completed" despite running into errors [4255](https://github.com/longhorn/longhorn/issues/4255) - @mantissahz @chriscchien - [BUG] Error during backup process will be removed quickly without user knowing [1249](https://github.com/longhorn/longhorn/issues/1249) - @mantissahz @chriscchien ### Misc - [TASK] Revert "Disable Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly for RWX volumes" [6838](https://github.com/longhorn/longhorn/issues/6838) - @derekbit @roger-ryao ## Contributors - @ChanYiLin - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @mantissahz - @nitendra-suse - @roger-ryao - @shuo-wu - @smallteeths - @votdev - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.3.md ================================================ ## Release Note ### **v1.5.3 released!** 🎆 This release focuses on resolving a regression issue from v1.5.2 related to volume encryption, along with a few other fixes and improvements. Please try it and provide feedback. Thanks for all the contributions! > For the definition of stable or latest release, please check [here](https://github.com/longhorn/longhorn#releases). ## Installation > **Please ensure your Kubernetes cluster is at least v1.21 before installing v1.5.3.** Longhorn supports three installation ways including Rancher App Marketplace, Kubectl, and Helm. Follow the installation instructions [here](https://longhorn.io/docs/1.5.3/deploy/install/). ## Upgrade > **Please read the [important notes](https://longhorn.io/docs/1.5.3/deploy/important-notes/) first and ensure your Kubernetes cluster is at least v1.21 before upgrading to Longhorn v1.5.3 from v1.4.x/v1.5.x, which are only supported source versions.** Follow the upgrade instructions [here](https://longhorn.io/docs/1.5.3/deploy/upgrade/). ## Deprecation & Incompatibilities N/A ## Known Issues after Release Please follow up on [here](https://github.com/longhorn/longhorn/wiki/Outstanding-Known-Issues-of-Releases) about any outstanding issues found after this release. ## Resolved Issues ### Improvement - [IMPROVEMENT] Add PVC namespace to longhorn_volume metrics [7077](https://github.com/longhorn/longhorn/issues/7077) - @mantissahz @roger-ryao @antoninferrand ### Resilience - [BUG] A race after a node reboot leads to I/O errors with migratable volumes [6961](https://github.com/longhorn/longhorn/issues/6961) - @yangchiu @ejweber ### Bug - [BUG] Share manager unmount/unexport RWX volume timing issue [7106](https://github.com/longhorn/longhorn/issues/7106) - @yangchiu @derekbit - [BUG] `backing-image-manager-` hostPath selection exception [7062](https://github.com/longhorn/longhorn/issues/7062) - @ChanYiLin @nitendra-suse - [BUG] Failing to mount encrypted volumes v1.5.2 [7045](https://github.com/longhorn/longhorn/issues/7045) - @derekbit @nitendra-suse - [BUG] Fix RWX volume mount option typo in v1.5.x [7104](https://github.com/longhorn/longhorn/issues/7104) - @yangchiu @derekbit - [BUG] Upgrade from v1.5.2 to v1.5.3-rc1 failed if there's an attached v1 volume and a detached v2 volume [7094](https://github.com/longhorn/longhorn/issues/7094) - @derekbit ## Contributors - @ChanYiLin - @antoninferrand - @derekbit - @ejweber - @innobead - @mantissahz - @nitendra-suse - @roger-ryao - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.4.md ================================================ ## Longhorn v1.5.4 Release Notes This latest stable version of Longhorn 1.5 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. > **Note:** > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.5.4.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.5.4/deploy/install/) in the Longhorn documentation. ## Upgrade **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.4.x to v1.5.4.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.5.4/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.5.4/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Highlight - [BACKPORT][v1.5.4][FEATURE] Add a new settings that allows Longhorn to evict replicas automatically when a node is drained [7421](https://github.com/longhorn/longhorn/issues/7421) - @yangchiu @ejweber ### Improvement - [BACKPORT][v1.5.4][IMPROVEMENT] Use HEAD instead of a GET to fetch the `Content-Length` of an resource via URL [7981](https://github.com/longhorn/longhorn/issues/7981) - @votdev @roger-ryao - [BACKPORT][v1.5.4][IMPROVEMENT] Change support-bundle-manager image pull policy to PullIfNotPresent [7999](https://github.com/longhorn/longhorn/issues/7999) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.4][FEATURE] Update base image of Longhorn components to BCI 15.5 [7134](https://github.com/longhorn/longhorn/issues/7134) - @nitendra-suse - [BACKPORT][v1.5.4]Allow to set mount options for storageclass via values.yaml in helm chart [7593](https://github.com/longhorn/longhorn/issues/7593) - @ChanYiLin @mantissahz - [BACKPORT][v1.5.4][IMPROVEMENT] Remove startup probe of CSI driver after liveness probe conn fix ready [7933](https://github.com/longhorn/longhorn/issues/7933) - @ejweber @chriscchien - [IMPROVEMENT] Make environment_check look for a global default K8s priority class in those releases that it affects. [7831](https://github.com/longhorn/longhorn/issues/7831) - @mantissahz @james-munson - [BACKPORT][v1.5.4][IMPROVEMENT] Allow deployment of Prometheus ServiceMonitor with the Longhorn helm chart [7864](https://github.com/longhorn/longhorn/issues/7864) - @mantissahz @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] Remove unused process manager connection in longhorn-manager [7785](https://github.com/longhorn/longhorn/issues/7785) - @derekbit @roger-ryao - [BACKPORT][v1.5.4][IMPROVEMENT] Clean up backup target in IM-R pod if the backup target setting is unset [7145](https://github.com/longhorn/longhorn/issues/7145) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] BackingImage should be compressed when downloading and use the name as filename instead of UUID [7397](https://github.com/longhorn/longhorn/issues/7397) - @ChanYiLin @roger-ryao - [BACKPORT][v1.5.4][IMPROVEMENT] Automatically remount read-only RWO volume to read-write [7500](https://github.com/longhorn/longhorn/issues/7500) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] deploy: driver deployer shouldn't cleanup previous deployment if Kubernetes version changes [7345](https://github.com/longhorn/longhorn/issues/7345) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.5.4][IMPROVEMENT] Only restarts pods with volumes in the unexpected Read-Only state [7729](https://github.com/longhorn/longhorn/issues/7729) - @yangchiu @ChanYiLin - [BACKPORT][v1.5.4][IMPROVEMENT] Improve handling of 16TiB+ volumes with ext4 as the underlying file system [7429](https://github.com/longhorn/longhorn/issues/7429) - @mantissahz @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] Volumes: metrics for snapshots include (size and type: system vs user) [7725](https://github.com/longhorn/longhorn/issues/7725) - @c3y1huang @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] Improve the profiler of longhorn-engine for runtime profiling [7545](https://github.com/longhorn/longhorn/issues/7545) - @Vicente-Cheng @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] Don't crash the migration engine when kubelet restarts [7328](https://github.com/longhorn/longhorn/issues/7328) - @yangchiu @ejweber - [BACKPORT][v1.5.4][IMPROVEMENT] Upgrade CSI components to the latest patch release [7492](https://github.com/longhorn/longhorn/issues/7492) - @c3y1huang @roger-ryao - [BACKPORT][v1.5.4][IMPROVEMENT] Reject the last replica deletion if its volume.spec.deletionTimestamp is not set [7432](https://github.com/longhorn/longhorn/issues/7432) - @yangchiu @derekbit - [BACKPORT][v1.5.4][IMPROVEMENT] Upgrade support bundle kit version to v0.0.33 [7279](https://github.com/longhorn/longhorn/issues/7279) - @c3y1huang - [BACKPORT][v1.5.4][IMPROVEMENT] Review and simplify longhorn component image build [7162](https://github.com/longhorn/longhorn/issues/7162) - @ChanYiLin - [BACKPORT][v1.5.4][IMPROVEMENT] Replace deprecated grpc.WithInsecure [7364](https://github.com/longhorn/longhorn/issues/7364) - @c3y1huang - [BACKPORT][v1.5.4][IMPROVEMENT] Have a setting to disable snapshot purge for maintenance purpose [7265](https://github.com/longhorn/longhorn/issues/7265) - @ejweber @chriscchien - [BACKPORT][v1.5.4][IMPROVEMENT] Bypass upgrade when installing a fresh setup [7283](https://github.com/longhorn/longhorn/issues/7283) - @mantissahz @roger-ryao ### Bug - [BACKPORT][v1.5.4][BUG][v1.5.4-rc4] Test case test_backuptarget_available_during_engine_image_not_ready failed to wait for backup target available [8054](https://github.com/longhorn/longhorn/issues/8054) - @yangchiu @c3y1huang - [BACKPORT][v1.5.4][BUG] The activated DR volume do not contain the latest data. [7947](https://github.com/longhorn/longhorn/issues/7947) - @shuo-wu @roger-ryao - [BUG][v1.5.x] DR volume unable to be activated if the latest backup's been deleted [7997](https://github.com/longhorn/longhorn/issues/7997) - @yangchiu @shuo-wu - [BUG] Backup related test cases failed [7989](https://github.com/longhorn/longhorn/issues/7989) - @yangchiu @shuo-wu - [BACKPORT][v1.5.4][BUG][v1.5.x] Recurring job fails to create backup when volume detached [8015](https://github.com/longhorn/longhorn/issues/8015) - @yangchiu @mantissahz @PhanLe1010 @c3y1huang - [BACKPORT][v1.5.4][BUG] Deadlock for RWX volume if an error occurs in its share-manager pod [7186](https://github.com/longhorn/longhorn/issues/7186) - @ejweber @chriscchien - [BACKPORT][v1.5.4][BUG] Deadlock is possible in v1.6.0 instance manager [7941](https://github.com/longhorn/longhorn/issues/7941) - @ejweber @roger-ryao - [BACKPORT][v1.5.4][BUG] Longhorn may keep corrupted salvaged replicas and discard good ones [7801](https://github.com/longhorn/longhorn/issues/7801) - @ejweber @chriscchien - [BACKPORT][v1.5.4][BUG] Deadlock between volume migration and upgrade after Longhorn upgrade [7869](https://github.com/longhorn/longhorn/issues/7869) - @ejweber @chriscchien - [BACKPORT][v1.5.4][BUG] Executing fstrim while rebuilding causes IO errors [7867](https://github.com/longhorn/longhorn/issues/7867) - @ejweber @chriscchien - [BACKPORT][v1.5.4][BUG] BackingImage does not download URL correctly in some situation [7986](https://github.com/longhorn/longhorn/issues/7986) - @yangchiu - [BACKPORT][v1.5.4][BUG] The feature of auto remount read only volume not work on a single node cluster. [7844](https://github.com/longhorn/longhorn/issues/7844) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.4][BUG] Volumes stuck upgrading after 1.5.3 -> 1.6.0 upgrade. [7901](https://github.com/longhorn/longhorn/issues/7901) - @yangchiu @ejweber - [BUG][v1.5.4-rc1] V2 volume have engine upgrade option on UI after upgrade from v1.5.3 to v1.5.4-rc1 [7863](https://github.com/longhorn/longhorn/issues/7863) - @chriscchien @scures - [BACKPORT][v1.5.4][BUG] longhorn manager pod fails to start in container-based K3s [7848](https://github.com/longhorn/longhorn/issues/7848) - @ChanYiLin - [BACKPORT][v1.5.4][BUG] Relax S3 client retry intervals, for throttled requests [7098](https://github.com/longhorn/longhorn/issues/7098) - @mantissahz @chriscchien - [BACKPORT][v1.5.4][BUG][v1.6.0-rc1] Negative test case failed: Stop Volume Node Kubelet For More Than Pod Eviction Timeout While Workload Heavy Writing [7761](https://github.com/longhorn/longhorn/issues/7761) - @yangchiu @c3y1huang - [BACKPORT][v1.5.4][BUG] Volumes don't mount with mTLS enabled [7789](https://github.com/longhorn/longhorn/issues/7789) - @derekbit @roger-ryao - [BACKPORT][v1.5.4][BUG] supportbundle/kubelet.log empty in k3s environment [7123](https://github.com/longhorn/longhorn/issues/7123) - @c3y1huang @chriscchien @roger-ryao - [BUG] v1.5.x/v1.4.x BackingImage download fails if URL has query parameters [7822](https://github.com/longhorn/longhorn/issues/7822) - @ChanYiLin @mantissahz - [BACKPORT][v1.5.4][BUG] Metric totalVolumeSize and totalVolumeActualSize incorrect due to v2 volume counts [7392](https://github.com/longhorn/longhorn/issues/7392) - @c3y1huang @chriscchien - [BACKPORT][v1.5.4][BUG] Continuously auto-balancing replicas when zone does not have enough space [7306](https://github.com/longhorn/longhorn/issues/7306) - @c3y1huang @chriscchien - [BACKPORT][v1.5.4][BUG] Unable to list backups when backuptarget resource is picked up by a cordoned node [7621](https://github.com/longhorn/longhorn/issues/7621) - @mantissahz @c3y1huang - [BUG] The wrong template in default-setting.yaml of the Longhorn chart in v1.5 and v1.4 [7459](https://github.com/longhorn/longhorn/issues/7459) - @mantissahz @roger-ryao - [BACKPORT][v1.5.4][BUG] Failed to `check_volume_data` after volume engine upgrade/migration [7402](https://github.com/longhorn/longhorn/issues/7402) - @PhanLe1010 @chriscchien - [BACKPORT][v1.5.4][BUG] Volume conditions are not represented in the UI for v1.4.x and newer [7242](https://github.com/longhorn/longhorn/issues/7242) - @m-ildefons @roger-ryao - [BACKPORT][v1.5.4][BUG] Confusing logging when trying to attach a new volume with no scheduled replicas [7245](https://github.com/longhorn/longhorn/issues/7245) - @ejweber @roger-ryao - [BACKPORT][v1.5.4][BUG] Environment check script claims success when kubectl fails. [7216](https://github.com/longhorn/longhorn/issues/7216) - @james-munson @roger-ryao - [BACKPORT][v1.5.4][BUG] Backup volume attachment tickets might not be cleaned up after completion. [7604](https://github.com/longhorn/longhorn/issues/7604) - @james-munson @chriscchien - [BACKPORT][v1.5.4][BUG][v1.6.0-rc1] Some Longhorn resources remaining after longhorn-uninstall job completed [7663](https://github.com/longhorn/longhorn/issues/7663) - @yangchiu @PhanLe1010 - [BACKPORT][v1.5.4][BUG] Backing Image Data Inconsistency if it's Exported from a Backing Image Backed Volume [7701](https://github.com/longhorn/longhorn/issues/7701) - @yangchiu @ChanYiLin - [BACKPORT][v1.5.4][BUG] CSI components CrashLoopBackOff, failed to connect to unix://csi/csi.sock after cluster restart [7426](https://github.com/longhorn/longhorn/issues/7426) - @ejweber @roger-ryao - [BACKPORT][v1.5.4][BUG] Volume could not be remounted after engine process killed [7772](https://github.com/longhorn/longhorn/issues/7772) - @ChanYiLin @shuo-wu @roger-ryao - [BACKPORT][v1.5.4][BUG] Enabling replica-auto-balance tries to replicate to disabled nodes causing lots of errors in the logs and in the UI [7275](https://github.com/longhorn/longhorn/issues/7275) - @yangchiu @c3y1huang - [BACKPORT][v1.5.4][BUG] `allow-collecting-longhorn-usage-metrics` setting is missing from chart settings [7250](https://github.com/longhorn/longhorn/issues/7250) - @ChanYiLin @roger-ryao - [BACKPORT][v1.5.4][BUG] When disabling revision counter, salvaging a faulty volume not work as expected [7732](https://github.com/longhorn/longhorn/issues/7732) - @james-munson @roger-ryao - [BACKPORT][v1.5.4][BUG] During volume live engine upgrade, delete replica with old engine image will make volume degraded forever [7334](https://github.com/longhorn/longhorn/issues/7334) - @PhanLe1010 @chriscchien - [BACKPORT][v1.5.4][BUG] Uninstallation job stuck forever if the MutatingWebhookConfigurations or ValidatingWebhookConfigurations already deleted [7658](https://github.com/longhorn/longhorn/issues/7658) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.5.4][BUG] Volume encryption doesn't work on Amazon Linux 2 [7165](https://github.com/longhorn/longhorn/issues/7165) - @derekbit @chriscchien - [BACKPORT][v1.5.4][BUG] Rancher cannot import longhorn 1.5 charts due to "error converting YAML to JSON: yaml: line 699: did not find expected key" [7776](https://github.com/longhorn/longhorn/issues/7776) - @mantissahz @PhanLe1010 - [BACKPORT][v1.5.4][BUG] Delete kubernetes node did not remove `node.longhorn.io` [7538](https://github.com/longhorn/longhorn/issues/7538) - @ejweber @chriscchien - [BACKPORT][v1.5.4][BUG] backingimage download server error [7381](https://github.com/longhorn/longhorn/issues/7381) - @scures @roger-ryao - [BACKPORT][v1.5.4][BUG] Longhorn-manager does not deploy CSI driver when integrated with linkerd service mesh [7391](https://github.com/longhorn/longhorn/issues/7391) - @yangchiu @mantissahz - [BACKPORT][v1.5.4][BUG] Helm2 install error: 'lookup' function not defined in validate-psp-install.yaml [7435](https://github.com/longhorn/longhorn/issues/7435) - @roger-ryao - [BACKPORT][v1.5.4][BUG] Warning events are being spammed by Longhorn - CRD [7309](https://github.com/longhorn/longhorn/issues/7309) - @m-ildefons @roger-ryao - [BACKPORT][v1.5.4][BUG] Persistent volume is not ready for workloads [7314](https://github.com/longhorn/longhorn/issues/7314) - @james-munson @roger-ryao - [BACKPORT][v1.5.4][BUG] Download backing image failed with HTTP 502 error if Storage Network configured [7239](https://github.com/longhorn/longhorn/issues/7239) - @ChanYiLin @roger-ryao - [BACKPORT][v1.5.4][BUG] Errors found by static checker in volume controller [7269](https://github.com/longhorn/longhorn/issues/7269) - @m-ildefons ### Misc - [TASK] Update v1.5.x Longhorn components vendor dependencies [8003](https://github.com/longhorn/longhorn/issues/8003) - @mantissahz @chriscchien - [BACKPORT][v1.5.4][TASK] Review why sessionAffinity: ClientIP is used in most services [7760](https://github.com/longhorn/longhorn/issues/7760) - @ejweber @roger-ryao - [BACKPORT][v1.5.4][TASK] Synchronize version of CSI components in longhorn/longhorn and longhorn/longhorn-manager [7378](https://github.com/longhorn/longhorn/issues/7378) - @c3y1huang @roger-ryao - [BACKPORT][v1.5.4][TASK] Bump the versions of dependent libs or components [7150](https://github.com/longhorn/longhorn/issues/7150) - @c3y1huang - [BACKPORT][v1.5.4][DOC] Fix erroneous value for default StorageMinimalAvailablePercentage setting. [7400](https://github.com/longhorn/longhorn/issues/7400) - @james-munson ## Contributors - @ChanYiLin - @PhanLe1010 - @Vicente-Cheng - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @m-ildefons - @mantissahz - @nitendra-suse - @roger-ryao - @scures - @shuo-wu - @votdev - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.5.5.md ================================================ ## Longhorn v1.5.5 Release Notes This latest stable version of Longhorn 1.5 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. > [!NOTE] > **For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases).** ## Installation > [!IMPORTANT] > **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.5.5.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.5.5/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] > **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.4.x or v1.5.x (< v1.5.5) to v1.5.5.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.5.5/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.5.5/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvements - [BACKPORT][v1.5.5][IMPROVEMENT] Cannot read/write to block volume when the container is run as non-root [8123](https://github.com/longhorn/longhorn/issues/8123) - @PhanLe1010 @chriscchien - [BACKPORT][v1.5.5][IMPROVEMENT] Do not terminate nfs-ganesha in share-manager pod after failing to access recovery backend [8347](https://github.com/longhorn/longhorn/issues/8347) - @derekbit @chriscchien - [BACKPORT][v1.5.5][IMPROVEMENT] Expose virtual size of qcow2 backing images [8321](https://github.com/longhorn/longhorn/issues/8321) - @shuo-wu @chriscchien - [BACKPORT][v1.5.5][IMPROVEMENT] Improve logging in CSI plugin when mount fails. [8286](https://github.com/longhorn/longhorn/issues/8286) - @james-munson @chriscchien - [BACKPORT][v1.5.5][IMPROVEMENT] Upgrade support bundle kit version to v0.0.36 [8161](https://github.com/longhorn/longhorn/issues/8161) - @c3y1huang @roger-ryao - [BACKPORT][v1.5.5][IMPROVEMENT] Improve environment_check script for NFS protocol bug and the host system self diagnosis [7972](https://github.com/longhorn/longhorn/issues/7972) - @james-munson @roger-ryao ### Bug Fixes - Security issues in latest longhorn docker images [8372](https://github.com/longhorn/longhorn/issues/8372) - @c3y1huang @chriscchien - [BACKPORT][v1.5.5][BUG] Backup marked as "completed" cannot be restored, gzip: invalid header [8378](https://github.com/longhorn/longhorn/issues/8378) - @derekbit @chriscchien - [BACKPORT][v1.5.5][BUG][v1.6.0-rc1] Failed to run instance-manager in storage network environment [8305](https://github.com/longhorn/longhorn/issues/8305) - @yangchiu @ejweber - [BACKPORT][v1.5.5][BUG] Replica rebuild failed [8257](https://github.com/longhorn/longhorn/issues/8257) - @shuo-wu @chriscchien - [BACKPORT][v1.5.5][BUG] longhorn manager pod fails to start in container-based K3s [7948](https://github.com/longhorn/longhorn/issues/7948) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.5][BUG] persistence.removeSnapshotsDuringFilesystemTrim Helm variable is unreferenced [7951](https://github.com/longhorn/longhorn/issues/7951) - @ejweber @roger-ryao - [BACKPORT][v1.5.5][BUG] Failed to restore a backup to file by the scripts/restore-backup-to-file.sh with a CIFS backup target. [8127](https://github.com/longhorn/longhorn/issues/8127) - @mantissahz @roger-ryao - [BACKPORT][v1.5.5][BUG] Longhorn api-server PUT request rate [8153](https://github.com/longhorn/longhorn/issues/8153) - @ejweber @roger-ryao - [BACKPORT][v1.5.5][BUG] A replica may be incorrectly scheduled to a node with an existing failed replica [8116](https://github.com/longhorn/longhorn/issues/8116) - @ejweber @chriscchien - [BACKPORT][v1.5.5][BUG] potential risk to unmap a negative number [8236](https://github.com/longhorn/longhorn/issues/8236) - @Vicente-Cheng @roger-ryao - [BACKPORT][v1.5.5][BUG] Use config map to update `default-replica-count` won't apply to `default-replica-count.definition.default` if the value equal to current `default-replica-count.value` [8135](https://github.com/longhorn/longhorn/issues/8135) - @james-munson @chriscchien - [BACKPORT][v1.5.5][BUG] LH manager reboots due to the webhook is not ready [8036](https://github.com/longhorn/longhorn/issues/8036) - @ChanYiLin @chriscchien - [BACKPORT][v1.5.5][BUG] Can't use longhorn with Generic ephemeral volumes [8201](https://github.com/longhorn/longhorn/issues/8201) - @ejweber @roger-ryao - [BACKPORT][v1.5.5][BUG] Volume cannot attach because of the leftover non-empty volume.status.PendingNodeID after upgrading Longhorn [7996](https://github.com/longhorn/longhorn/issues/7996) - @james-munson @roger-ryao - [BACKPORT][v1.5.5][BUG] no Pending workload pods for volume xxx to be mounted [8082](https://github.com/longhorn/longhorn/issues/8082) - @c3y1huang @roger-ryao ### Miscellaneous - [TASK] update go-iscsi-helper and go-common-lib in v1.5.x [7960](https://github.com/longhorn/longhorn/issues/7960) - @ChanYiLin - [BACKPORT][v1.5.5][REFACTOR] move mount point check function to common lib [8125](https://github.com/longhorn/longhorn/issues/8125) - @ChanYiLin @roger-ryao ## Contributors - @ChanYiLin - @PhanLe1010 - @Vicente-Cheng - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @mantissahz - @roger-ryao - @shuo-wu - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.6.0.md ================================================ ## Longhorn v1.6.0 Release Notes This latest version of Longhorn introduces several features, enhancements, and bug fixes that are intended to improve system quality and the overall user experience. Highlights include new V2 Data Engine features, platform-agnostic deployment, node maintenance, and improvements to stability, performance, and resilience. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. > **Note:** > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Primary Highlights ### New V2 Data Engine Features Although the V2 Data Engine is still considered a preview feature in this release, the core functions have been significantly enhanced. For example, you can now seamlessly perform volume backup and restore operations between the V1 and V2 Data Engines, paving the way for volume migration between the two data engines in the future. - [Volume Snapshot and Revert](https://github.com/longhorn/longhorn/issues/6137) - [Volume Backup and Restore](https://github.com/longhorn/longhorn/issues/6138) - [Separate Data Plane for v1 and v2 Data Engines](https://github.com/longhorn/longhorn/issues/7015) - [ARM64 Support](https://github.com/longhorn/longhorn/issues/6021) The Longhorn team will continue to develop features for the V1 Data Engine and to prepare the V2 Data Engine for use in all types of environments. ### Platform-Agnostic Deployment Longhorn is designed to seamlessly operate on general-purpose Linux distributions, and on certain container-optimized systems such as SLE Micro. In response to numerous requests, v1.6.0 was enhanced to allow installation of Longhorn components on [Talos](https://www.talos.dev/), which is a secure, immutable, and minimal Kubernetes OS. v1.6.0 also includes OKD support, which was contributed by community member @ArthurVardevanyan. - [Talos Support](https://github.com/longhorn/longhorn/issues/3161) - [OKD (OpenShift Origin) Support](https://github.com/longhorn/longhorn/issues/1831) The Longhorn team is committed to making Longhorn an adaptive storage solution and anticipates receiving feedback regarding your preferred platforms. ### Space Efficiency Starting with v1.6.0, Longhorn allows you to configure the maximum snapshot count and the maximum aggregate snapshot size for all volumes and for specific volumes. Both settings, whether applied globally or individually, aid in space estimation and management. Earlier Longhorn versions do not provide mechanisms for controlling or predicting the quantity and size of volume snapshots. - [Snapshot Space Management](https://github.com/longhorn/longhorn/issues/6563) ### GitOps Friendly Longhorn has been validated with popular GitOps solutions, including [Flux](https://github.com/longhorn/longhorn/issues/6343), [Argo CD](https://github.com/longhorn/longhorn/issues/6434), and [Fleet](https://github.com/longhorn/longhorn/issues/6935). Future releases will include enhancements that further solidify Longhorn's status as a GitOps-aware storage solution. ### Data Protection Longhorn now supports [block volume encryption](https://github.com/longhorn/longhorn/issues/4883), which is particularly beneficial in virtualization use cases such as Harvester and KubeVirt. ### Node Maintenance v1.6.0 includes two new [node drain policy options](https://longhorn.io/docs/1.6.0/references/settings/#node-drain-policy): *Block For Eviction* and *Block For Eviction If Contains Last Replica*. Both options allow automatic eviction and relocation of healthy replicas from draining nodes (before the nodes are cordoned). The Longhorn team recommends enabling these options only during planned maintenance to minimize impact on data movement. For more information about the advantages and disavantages of all options, see [Node Drain Policy Recommendations](../../volumes-and-nodes/maintenance/#node-drain-policy-recommendations) in the Longhorn documentation. ### Backing Image Management Longhorn now allows you to [create and restore backups of backing images](https://github.com/longhorn/longhorn/issues/4165), which can streamline the management of backing images across clusters. This feature is particularly beneficial in virtualization use cases such as Harvester and KubeVirt. ## Installation **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.6.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.6.0/deploy/install/) in the Longhorn documentation. ## Upgrade **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.5.x to v1.6.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.6.0/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.6.0/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Highlights - [FEATURE] Longhorn snapshot space management [6563](https://github.com/longhorn/longhorn/issues/6563) - @FrankYang0529 @yangchiu - [FEATURE] v2 data engine volume snapshot and revert [6137](https://github.com/longhorn/longhorn/issues/6137) - @shuo-wu @roger-ryao - [FEATURE] Support eventual danger zone setting update [7173](https://github.com/longhorn/longhorn/issues/7173) - @mantissahz @chriscchien - [FEATURE] Engine upgrade enforcement [5842](https://github.com/longhorn/longhorn/issues/5842) - @yangchiu @mantissahz @c3y1huang - [FEATURE] Selective V2 Data Engine Activation [7015](https://github.com/longhorn/longhorn/issues/7015) - @derekbit @chriscchien @roger-ryao - [FEATURE] Have default priorityClass to prevent unexpected longhorn pods eviction [6528](https://github.com/longhorn/longhorn/issues/6528) - @mantissahz @chriscchien - [FEATURE] v2 volume supports volume backup/restore [6138](https://github.com/longhorn/longhorn/issues/6138) - @yangchiu @derekbit - [IMPROVEMENT] Remove or Change Helm pre-upgrade hook to support ArgoCD [6415](https://github.com/longhorn/longhorn/issues/6415) - @mantissahz - [FEATURE] Restore BackingImage for BackupVolume in a new cluster [4165](https://github.com/longhorn/longhorn/issues/4165) - @ChanYiLin @roger-ryao - [FEATURE] Talos support [3161](https://github.com/longhorn/longhorn/issues/3161) - @yangchiu @c3y1huang - [FEATURE] Support v2 volume on ARM64 platform [6021](https://github.com/longhorn/longhorn/issues/6021) - @derekbit @chriscchien @roger-ryao - [IMPROVEMENT] Add a new setting that allows Longhorn to evict replicas automatically when a node is drained [2238](https://github.com/longhorn/longhorn/issues/2238) - @ejweber @chriscchien - [FEATURE] Add linear dm device on the top of v2 volume [7357](https://github.com/longhorn/longhorn/issues/7357) - @derekbit @chriscchien - [FEATURE] Support Encryption for VolumeMode Block [4883](https://github.com/longhorn/longhorn/issues/4883) - @derekbit @roger-ryao - [TASK] Add install/upgrade longhorn by gitops (flux) pipeline [6343](https://github.com/longhorn/longhorn/issues/6343) - @yangchiu - [FEATURE] OKD/Openshift support [1831](https://github.com/longhorn/longhorn/issues/1831) - @mantissahz @ArthurVardevanyan @roger-ryao ### Features - [UI][FEATURE] Longhorn snapshot space management [7522](https://github.com/longhorn/longhorn/issues/7522) - @yangchiu @scures - [FEATURE] RWX volume supports different NFS version (4.2) and mount options [7638](https://github.com/longhorn/longhorn/issues/7638) - @james-munson - [FEATURE] Introduce `upgradeVersionCheck` to decide version upgrade enforcement [7539](https://github.com/longhorn/longhorn/issues/7539) - @mantissahz @chriscchien - [FEATURE] v2 volume replica management [5420](https://github.com/longhorn/longhorn/issues/5420) - @DamiaSan - [FEATURE] Update nfs-genesha to 5.x for share manager [6000](https://github.com/longhorn/longhorn/issues/6000) - @james-munson @chriscchien - [FEATURE] Allow to set mount options for storageclass via values.yaml in helm chart [7351](https://github.com/longhorn/longhorn/issues/7351) - @ChanYiLin @chriscchien - [FEATURE] Flush on-the-fly IOs in the queue before snapshotting [5648](https://github.com/longhorn/longhorn/issues/5648) - @DamiaSan - [FEATURE] Update base image of Longhorn components to BCI 15.5 [6206](https://github.com/longhorn/longhorn/issues/6206) - @nitendra-suse - [FEATURE] Customize MaxRecurringJobRetain [5713](https://github.com/longhorn/longhorn/issues/5713) - @mantissahz @chriscchien - [FEATURE] Replica rebuild over SPDK [5216](https://github.com/longhorn/longhorn/issues/5216) - @shuo-wu @DamiaSan - [FEATURE] Allow kubectl drain to stop manually attached volumes [6978](https://github.com/longhorn/longhorn/issues/6978) - @ChanYiLin @chriscchien - [FEATURE] Single Node Disk affinity [3823](https://github.com/longhorn/longhorn/issues/3823) - @ejweber @roger-ryao - [FEATURE] Storage network support for Multus v4.0 thick-plugin [5048](https://github.com/longhorn/longhorn/issues/5048) - @c3y1huang @chriscchien - [FEATURE] Add disk status prometheus metrics [6858](https://github.com/longhorn/longhorn/issues/6858) - @c3y1huang @chriscchien - [FEATURE] Add a brand new/empty bdev with WriteOnly mode to the RAID1 bdev [5865](https://github.com/longhorn/longhorn/issues/5865) - @DamiaSan - [FEATURE] Add a script to identify the valid volumes to recover given s3 backup url and secret [1523](https://github.com/longhorn/longhorn/issues/1523) - @weizhe0422 - [FEATURE] Pause IO when raid1 bdev snapshotting [5421](https://github.com/longhorn/longhorn/issues/5421) - @DamiaSan - [FEATURE] Change the replica selector behavior so that an absent selector is able to select nodes without a TAG [4826](https://github.com/longhorn/longhorn/issues/4826) - @ChanYiLin @roger-ryao - [FEATURE] Helm Chart make loglevel configurable [3655](https://github.com/longhorn/longhorn/issues/3655) - @mantissahz ### Improvements - [IMPROVEMENT] Use ensureFolderPath rather than ensureMount for checking folder path in CSI plugin [7784](https://github.com/longhorn/longhorn/issues/7784) - @derekbit @roger-ryao - [IMPROVEMENT] Remove unused process manager connection in longhorn-manager [7783](https://github.com/longhorn/longhorn/issues/7783) - @derekbit @chriscchien - [IMPROVEMENT] Revert RWX volume back to NFS v4.1 [7741](https://github.com/longhorn/longhorn/issues/7741) - @yangchiu @james-munson - [IMPROVEMENT] Remove static sessionAffinity: ClientIP set in most services if not required [7399](https://github.com/longhorn/longhorn/issues/7399) - @yangchiu @ejweber - [IMPROVEMENT] Automatically remount read-only RWO volume to read-write [6386](https://github.com/longhorn/longhorn/issues/6386) - @ChanYiLin @chriscchien - [IMPROVEMENT] Only restarts pods with volumes in the unexpected Read-Only state [7728](https://github.com/longhorn/longhorn/issues/7728) - @ChanYiLin @chriscchien - [IMPROVEMENT] Volumes: metrics for snapshots include (size and type: system vs user) [5869](https://github.com/longhorn/longhorn/issues/5869) - @c3y1huang @chriscchien - [IMPROVEMENT] Have a clear message when reverting the parent of a volume-head snapshot for a v2 volume [7630](https://github.com/longhorn/longhorn/issues/7630) - @derekbit @chriscchien - [IMPROVEMENT] Flooding error messages `failed to sync setting for.....` [7654](https://github.com/longhorn/longhorn/issues/7654) - @mantissahz @chriscchien - [IMPROVEMENT] Enhance the code quality in the instance-manager instance and disk gRPC server methods. [7628](https://github.com/longhorn/longhorn/issues/7628) - @derekbit - [IMPROVEMENT] Increase the hugepage size for spdk_tgt to 2GiB [7606](https://github.com/longhorn/longhorn/issues/7606) - @derekbit @chriscchien - [IMPROVEMENT] Reject DR volume creation for v2 volume [7627](https://github.com/longhorn/longhorn/issues/7627) - @derekbit @roger-ryao - [IMPROVEMENT] Do not use `--force` for dmsetup remove command [7615](https://github.com/longhorn/longhorn/issues/7615) - - [IMPROVEMENT] Update nvme-cli to v2.7.1 in instance-manager pod [7609](https://github.com/longhorn/longhorn/issues/7609) - @derekbit - [IMPROVEMENT] Prevent from complains in spdk_tgt when deleting a v2 volume [7568](https://github.com/longhorn/longhorn/issues/7568) - @yangchiu @derekbit @roger-ryao - [IMPROVEMENT] Expose actual size of a logical volume [5947](https://github.com/longhorn/longhorn/issues/5947) - @derekbit @shuo-wu @chriscchien @DamiaSan - [IMPROVEMENT] UI backup restoration supports v1 and v2 `Data Engine` [6597](https://github.com/longhorn/longhorn/issues/6597) - @derekbit @scures @roger-ryao - [IMPROVEMENT][UI] Display v2 volume actual size [7524](https://github.com/longhorn/longhorn/issues/7524) - @derekbit @chriscchien - [IMPROVEMENT] Recreate instance manager pod for v2 volume when `spdk_tgt` is dead [7551](https://github.com/longhorn/longhorn/issues/7551) - @derekbit @chriscchien - [IMPROVEMENT] Add reserve storage percentage of nodes setting in helm chart [5958](https://github.com/longhorn/longhorn/issues/5958) - @mantissahz @roger-ryao - [IMPROVEMENT] Reconcile engine/replica instance state of v2 volume like v1 volume [7326](https://github.com/longhorn/longhorn/issues/7326) - @derekbit @chriscchien - [IMPROVEMENT] Improve handling of 16TiB+ volumes with ext4 as the underlying file system [7423](https://github.com/longhorn/longhorn/issues/7423) - @mantissahz @chriscchien - [IMPROVEMENT] Rename backendStoreDriver to dataEngin in instance-manager and associated components [7480](https://github.com/longhorn/longhorn/issues/7480) - @yangchiu @derekbit - [IMPROVEMENT][UI] Validate volume creation according to the enabled data engines [7505](https://github.com/longhorn/longhorn/issues/7505) - @derekbit @chriscchien - [IMPROVEMENT] Add guaranteed instanceManager CPU setting for v2 volume [7361](https://github.com/longhorn/longhorn/issues/7361) - @derekbit @roger-ryao - [IMPROVEMENT] Support backup list if there is only v2-data-engine enabled [7486](https://github.com/longhorn/longhorn/issues/7486) - @derekbit @chriscchien - [IMPROVEMENT] Upgrade CSI components to the latest patch release [7384](https://github.com/longhorn/longhorn/issues/7384) - @c3y1huang @roger-ryao - [IMPROVEMENT] Add global setting for enable v1 or v2 volume support [7095](https://github.com/longhorn/longhorn/issues/7095) - @yangchiu @derekbit - [IMPROVEMENT] Blindly stop raid bdev exposure before exposing it for V2 volume [7324](https://github.com/longhorn/longhorn/issues/7324) - @yangchiu @derekbit @roger-ryao - [IMPROVEMENT] instance-managers for v1 and v2 volumes respectively [6984](https://github.com/longhorn/longhorn/issues/6984) - @yangchiu @derekbit - [IMPROVEMENT] BackingImage should be compressed when downloading and use the name as filename instead of UUID [7295](https://github.com/longhorn/longhorn/issues/7295) - @ChanYiLin @chriscchien - [IMPROVEMENT] Reject the creation of encrypted v2 volume in validating webhook [7404](https://github.com/longhorn/longhorn/issues/7404) - @derekbit @chriscchien - [IMPROVEMENT] Longhorn-engine processes should refuse to serve requests not intended for them [5845](https://github.com/longhorn/longhorn/issues/5845) - @ejweber @chriscchien - [IMPROVEMENT] Collect v2 Data Engine related info for the usage metrics [6033](https://github.com/longhorn/longhorn/issues/6033) - @c3y1huang @chriscchien - [IMPROVEMENT] Review and simplify longhorn component image build [5911](https://github.com/longhorn/longhorn/issues/5911) - @ChanYiLin @chriscchien - [IMPROVEMENT] Gracefully shut down spdk_tgt [7263](https://github.com/longhorn/longhorn/issues/7263) - @derekbit @chriscchien - [IMPROVEMENT] Reject the last replica deletion if its volume.spec.deletionTimestamp is not set [7372](https://github.com/longhorn/longhorn/issues/7372) - @yangchiu @derekbit - [IMPROVEMENT] add build script to generate gRPC related code more convenient [6973](https://github.com/longhorn/longhorn/issues/6973) - @Vicente-Cheng - [IMPROVEMENT] Upgrade support bundle kit version to v0.0.33 [7277](https://github.com/longhorn/longhorn/issues/7277) - @c3y1huang - [IMPROVEMENT] Upgrade CSI sidecar components version [6916](https://github.com/longhorn/longhorn/issues/6916) - @c3y1huang @roger-ryao - [IMPROVEMENT] Have a setting to disable snapshot purge for maintenance purpose [7075](https://github.com/longhorn/longhorn/issues/7075) - @ejweber @roger-ryao - [IMPROVEMENT] Don't crash the migration engine when kubelet restarts [7302](https://github.com/longhorn/longhorn/issues/7302) - @ejweber @chriscchien - [IMPROVEMENT] deploy: driver deployer shouldn't cleanup previous deployment if Kubernetes version changes [5474](https://github.com/longhorn/longhorn/issues/5474) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Replace deprecated grpc.WithInsecure [7291](https://github.com/longhorn/longhorn/issues/7291) - @c3y1huang - [IMPROVEMENT] Allow deployment of Prometheus ServiceMonitor with the Longhorn helm chart [7041](https://github.com/longhorn/longhorn/issues/7041) - @mantissahz @chriscchien - [IMPROVEMENT] Disable CGO in longhorn components if not used [7135](https://github.com/longhorn/longhorn/issues/7135) - @derekbit - [IMPROVEMENT] Add test for longhorn-spdk-engine [6060](https://github.com/longhorn/longhorn/issues/6060) - @shuo-wu - [IMPROVEMENT] Thread-safe SPDK JSON client [6106](https://github.com/longhorn/longhorn/issues/6106) - @shuo-wu - [IMPROVEMENT] Bypass upgrade when installing a fresh setup [6988](https://github.com/longhorn/longhorn/issues/6988) - @mantissahz @roger-ryao - [IMPROVEMENT] Upgrade support bundle kit version to v0.0.32 [7152](https://github.com/longhorn/longhorn/issues/7152) - @c3y1huang @chriscchien - [IMPROVEMENT] Support custom options for network filesystems for backup [6608](https://github.com/longhorn/longhorn/issues/6608) - @james-munson @roger-ryao - [IMPROVEMENT] Global setting `default-data-path` supports block device [7234](https://github.com/longhorn/longhorn/issues/7234) - @derekbit @chriscchien - [IMPROVEMENT] Clean up backup target in IM-R pod if the backup target setting is unset [5741](https://github.com/longhorn/longhorn/issues/5741) - @ChanYiLin @chriscchien - [IMPROVEMENT] Improve log level for resource update failure able to reconcile again [6843](https://github.com/longhorn/longhorn/issues/6843) - @PhanLe1010 @nitendra-suse - [IMPROVEMENT] Add missing volume settings to the default storage class [6496](https://github.com/longhorn/longhorn/issues/6496) - @james-munson - [IMPROVEMENT] High memory consumption of longhorn-manager pods since Longhorn v1.5 [6936](https://github.com/longhorn/longhorn/issues/6936) - @derekbit @roger-ryao - [IMPROVEMENT] Upgrade support bundle kit version to v0.0.29 [6922](https://github.com/longhorn/longhorn/issues/6922) - @c3y1huang @chriscchien - [IMPROVEMENT] Improve upgrade path and make it more solid [6294](https://github.com/longhorn/longhorn/issues/6294) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Use nvme-cli in instance-manager pod instead [6798](https://github.com/longhorn/longhorn/issues/6798) - @derekbit @chriscchien - [IMPROVEMENT] Add PVC namespace to longhorn_volume metrics [7077](https://github.com/longhorn/longhorn/issues/7077) - @mantissahz @roger-ryao @antoninferrand - [IMPROVEMENT] Don't log about inability to change settings that didn't change. [6812](https://github.com/longhorn/longhorn/issues/6812) - @james-munson @roger-ryao - [IMPROVEMENT] Consolidate the mounts in longhorn-manager and instance-manager [5883](https://github.com/longhorn/longhorn/issues/5883) - @ChanYiLin - [IMPROVEMENT] Make the timeout value of a filesystem-based backup store configurable [5723](https://github.com/longhorn/longhorn/issues/5723) - @ChanYiLin - [IMPROVEMENT] Unify logs with extra static info like module/method/function/line [5509](https://github.com/longhorn/longhorn/issues/5509) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Prevent Volume Provision if Related Backing Image Stuck in Ready-For-Trasfer State [6615](https://github.com/longhorn/longhorn/issues/6615) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Remove dummy services of each CSI sidecar if not required [6581](https://github.com/longhorn/longhorn/issues/6581) - @ejweber @roger-ryao - [IMPROVEMENT] Old kernel such as 3.10.0 set provisioning_mode to wrong value (writesame_16, disabled, full, ...) but not the correct value (unmap) so the trim feature doesn't work [6854](https://github.com/longhorn/longhorn/issues/6854) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Support both NFS `hard` and `soft` with custom `timeo` and `retrans` options for RWX volumes [6655](https://github.com/longhorn/longhorn/issues/6655) - @derekbit @roger-ryao - [IMPROVEMENT] Prevent unexpected engine creation [6682](https://github.com/longhorn/longhorn/issues/6682) - @PhanLe1010 @ejweber @roger-ryao - [IMPROVEMENT] Add pvc name to longhorn_volume metrics [5297](https://github.com/longhorn/longhorn/issues/5297) - @c3y1huang @nitendra-suse - [IMPROVEMENT] Replace `engineImage` field in CRDs with `image` [6647](https://github.com/longhorn/longhorn/issues/6647) - @derekbit @chriscchien - [IMPROVEMENT] Fix scheduling flooding logs [6019](https://github.com/longhorn/longhorn/issues/6019) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Avoid the accident deletion of longhorn settings [4984](https://github.com/longhorn/longhorn/issues/4984) - @ejweber @roger-ryao - [IMPROVEMENT] UI: making batch deletion dialog more readable [4080](https://github.com/longhorn/longhorn/issues/4080) - @smallteeths - [IMPROVEMENT] Upgrade Longhorn upgrade-responder server and build new Grafana dashboard [6368](https://github.com/longhorn/longhorn/issues/6368) - @PhanLe1010 - [IMPROVEMENT] Consider adding owner reference Backup/BackupVolume CR [5896](https://github.com/longhorn/longhorn/issues/5896) - @ChanYiLin - [IMPROVEMENT] Include /var/log/messages during the support-bundle syslog collection [6544](https://github.com/longhorn/longhorn/issues/6544) - @c3y1huang @roger-ryao - [IMPROVEMENT] UI Volume detail page still shows `Block Device` when `spec.disableFrontend` is true [6167](https://github.com/longhorn/longhorn/issues/6167) - @smallteeths @chriscchien - [IMPROVEMENT] Remove Longhorn engine path mismatch log [3786](https://github.com/longhorn/longhorn/issues/3786) - @c3y1huang @roger-ryao - [IMPROVEMENT] Provide more information for volume scheduling failure [6461](https://github.com/longhorn/longhorn/issues/6461) - @smallteeths @chriscchien - [IMPROVEMENT] Implement/fix the unit tests of Volume Attachment and volume controller [6005](https://github.com/longhorn/longhorn/issues/6005) - @PhanLe1010 @roger-ryao - [QUESTION] Repetitive warnings and errors in a new longhorn setup [6257](https://github.com/longhorn/longhorn/issues/6257) - @derekbit @c3y1huang @roger-ryao - [IMPROVEMENT] Make environment check script recognize iscsid.socket enable instead of iscsid.server only [5380](https://github.com/longhorn/longhorn/issues/5380) - @derekbit @roger-ryao ### Bug Fixes - [BUG] Volumes don't mount with mTLS enabled [7040](https://github.com/longhorn/longhorn/issues/7040) - @sfackler @derekbit @c3y1huang @ejweber @chriscchien - [BUG] Negative test case failed: Stop Volume Node Kubelet For More Than Pod Eviction Timeout While Workload Heavy Writing [7694](https://github.com/longhorn/longhorn/issues/7694) - @yangchiu @c3y1huang - [BUG] Rancher cannot import longhorn 1.5 charts due to "error converting YAML to JSON: yaml: line 699: did not find expected key" [7496](https://github.com/longhorn/longhorn/issues/7496) - @mantissahz @PhanLe1010 - [BUG] Volume could not be remounted after engine process killed [7751](https://github.com/longhorn/longhorn/issues/7751) - @yangchiu @ChanYiLin @shuo-wu - [BUG][v1.6.0-rc2] rwx volume failed to execute trim filesystem [7768](https://github.com/longhorn/longhorn/issues/7768) - @c3y1huang @roger-ryao - [BUG] Protect spdkClient in ReplicaCreate [7752](https://github.com/longhorn/longhorn/issues/7752) - @derekbit @chriscchien - [BUG] When disabling revision counter, salvaging a faulty volume not work as expected [7714](https://github.com/longhorn/longhorn/issues/7714) - @james-munson @roger-ryao - [BUG] Chart v1.6.0-rc2 is not synced between longhorn/longhorn and longhorn/charts [7743](https://github.com/longhorn/longhorn/issues/7743) - @innobead @chriscchien - [BUG] v2 Engine does not show the rebuilding replica mode [7718](https://github.com/longhorn/longhorn/issues/7718) - @shuo-wu - [BUG] Volume rebuilding never succeed after the first rebuilding failed [7723](https://github.com/longhorn/longhorn/issues/7723) - @shuo-wu @chriscchien - [BUG] Backing Image Data Inconsistency if it's Exported from a Backing Image Backed Volume [6899](https://github.com/longhorn/longhorn/issues/6899) - @ChanYiLin @chriscchien - [BUG] Remove v2 volume rebuild snapshot could cause volume stuck in detaching/faulted state [7573](https://github.com/longhorn/longhorn/issues/7573) - @yangchiu @shuo-wu - [BUG] Incompatible engine image kept in "deploying" state on master-head [7683](https://github.com/longhorn/longhorn/issues/7683) - @mantissahz @chriscchien - [BUG] Updating Taint Toleration is allowed if there are volumes attached [7675](https://github.com/longhorn/longhorn/issues/7675) - @yangchiu @mantissahz - [BUG] After v2 volume offline rebuilding, re-attached volume remains degraded [7574](https://github.com/longhorn/longhorn/issues/7574) - @yangchiu @shuo-wu - [BUG] Update settings GuaranteedInstanceManagerCPU and V2DataEngineGuaranteedInstanceManagerCPU separately [7676](https://github.com/longhorn/longhorn/issues/7676) - @mantissahz @chriscchien - [BUG] Uninstallation job stuck forever if the MutatingWebhookConfigurations or ValidatingWebhookConfigurations already deleted [7657](https://github.com/longhorn/longhorn/issues/7657) - @PhanLe1010 @roger-ryao - [BUG][v1.6.0-rc1] Some Longhorn resources remaining after longhorn-uninstall job completed [7645](https://github.com/longhorn/longhorn/issues/7645) - @yangchiu @PhanLe1010 - [BUG] Crypt device mapper of RWX volume with `migratable=true` is not cleaned up [7678](https://github.com/longhorn/longhorn/issues/7678) - @derekbit @chriscchien - [BUG] replica not rebuild in v1.6.0-dev if engine image is v1.4.x [7631](https://github.com/longhorn/longhorn/issues/7631) - @mantissahz @chriscchien - [BUG] Warning events are being spammed by Longhorn - CRD [7290](https://github.com/longhorn/longhorn/issues/7290) - @m-ildefons @roger-ryao - [BUG][v1.6.0-rc1] Error message in longhorn-uninstall job logs [7643](https://github.com/longhorn/longhorn/issues/7643) - @yangchiu @ChanYiLin - [BUG][v1.6.0-rc1] Failed to run instance-manager in storage network environment [7640](https://github.com/longhorn/longhorn/issues/7640) - @yangchiu @c3y1huang - [BUG] Volume with v1.5.x engine not worked well in v1.6.0-rc1 [7642](https://github.com/longhorn/longhorn/issues/7642) - @FrankYang0529 @chriscchien - [BUG] Unable to list backups when backuptarget resource is picked up by a cordoned node [7619](https://github.com/longhorn/longhorn/issues/7619) - @derekbit @c3y1huang @chriscchien - [BUG] Update the description of v2-data-engine setting [7655](https://github.com/longhorn/longhorn/issues/7655) - @derekbit - [BUG] Deleting instance-manager during restoring a v2 volume, the volume stuck in detaching state [7581](https://github.com/longhorn/longhorn/issues/7581) - @derekbit @chriscchien @roger-ryao - [BUG] Deleting instance-manager pod causes v2 volume stuck in attaching/detaching loop [7579](https://github.com/longhorn/longhorn/issues/7579) - @derekbit @roger-ryao - [BUG] After some v2 volume operations, v2 instance manager on a specific node somehow doesn't work [7608](https://github.com/longhorn/longhorn/issues/7608) - @yangchiu @derekbit - [BUG] Inconsistent behavior of snapshot list between v1 and v2 volume [7622](https://github.com/longhorn/longhorn/issues/7622) - @yangchiu @derekbit - [BUG] Fix and improve the offline rebuilding after introducing the SPDK snapshot feature [7596](https://github.com/longhorn/longhorn/issues/7596) - @shuo-wu @chriscchien - [BUG] Backup volume attachment tickets might not be cleaned up after completion. [6654](https://github.com/longhorn/longhorn/issues/6654) - @james-munson @chriscchien - [BUG] Correct the naming of v2 volume snapshot created after backup restoration [7577](https://github.com/longhorn/longhorn/issues/7577) - @derekbit @chriscchien - [BUG] Randomly failed to create volume with backing image [7543](https://github.com/longhorn/longhorn/issues/7543) - @yangchiu @ChanYiLin - [BUG] v2 volume becomes faulted and detached after deleting one replica during full restoration [7597](https://github.com/longhorn/longhorn/issues/7597) - @derekbit @chriscchien - [BUG] Creating volume randomly failed: failed to find a node that is ready and has the default engine image [7413](https://github.com/longhorn/longhorn/issues/7413) - @yangchiu @PhanLe1010 - [BUG] Delete error backup could cause v2 volume stuck in detaching/faulted state [7575](https://github.com/longhorn/longhorn/issues/7575) - @derekbit @roger-ryao - [BUG] Restore v2 volume stuck in detaching/faulted state if the backup is corrupted [7583](https://github.com/longhorn/longhorn/issues/7583) - @derekbit @chriscchien - [BUG] After upgrade to master-head, existing volume won't rebuild replica if one deleted, and the volume keeps healthy instead of degraded [7555](https://github.com/longhorn/longhorn/issues/7555) - @FrankYang0529 @yangchiu @derekbit - [BUG] Delete the backup during restoring a v2 volume from the backup, the restore volume will be detached and faulted [7584](https://github.com/longhorn/longhorn/issues/7584) - @derekbit - [BUG] Fix the failure of `test_basic.py:: test_volume_scheduling_failure` for v2 volumes [7570](https://github.com/longhorn/longhorn/issues/7570) - @derekbit @chriscchien - [BUG] Fix using deprecated option of `blockdev` command in go-spdk-helper [7567](https://github.com/longhorn/longhorn/issues/7567) - @derekbit - [BUG] Delete kubernetes node did not remove `node.longhorn.io` [7475](https://github.com/longhorn/longhorn/issues/7475) - @ejweber @chriscchien - [BUG] Failed to `check_volume_data` after volume engine upgrade/migration [7396](https://github.com/longhorn/longhorn/issues/7396) - @PhanLe1010 @james-munson @roger-ryao - [BUG] Failed RWX mount due to connection timeout still happening [7301](https://github.com/longhorn/longhorn/issues/7301) - @james-munson - [BUG] V2 volume is attached to a node first, the V1 volume will fails to attach. [7511](https://github.com/longhorn/longhorn/issues/7511) - @c3y1huang @roger-ryao - [BUG] v2 volume always displays engine upgrade available on UI [7489](https://github.com/longhorn/longhorn/issues/7489) - @scures - [BUG] Create volume(v1) faulted [7536](https://github.com/longhorn/longhorn/issues/7536) - @FrankYang0529 @chriscchien - [BUG] Persistent volume is not ready for workloads [6776](https://github.com/longhorn/longhorn/issues/6776) - @james-munson @roger-ryao - [BUG] Unable to create snapshot: cannot get engine client because it isn't deployed [7438](https://github.com/longhorn/longhorn/issues/7438) - @yangchiu @PhanLe1010 - [BUG] Deadlock for RWX volume if an error occurs in its share-manager pod [7183](https://github.com/longhorn/longhorn/issues/7183) - @derekbit @chriscchien - [BUG] Volume conditions are not represented in the UI for v1.4.x and newer [7241](https://github.com/longhorn/longhorn/issues/7241) - @m-ildefons @chriscchien - [BUG] backingimage download server error [7288](https://github.com/longhorn/longhorn/issues/7288) - @scures @roger-ryao - [BUG] CSI components CrashLoopBackOff, failed to connect to unix://csi/csi.sock after cluster restart [7116](https://github.com/longhorn/longhorn/issues/7116) - @yangchiu @ejweber - [BUG] Kubelet cannot finish terminating a pod that uses a PVC with volumeMode: Block when restarting the node [6919](https://github.com/longhorn/longhorn/issues/6919) - @PhanLe1010 @chriscchien - [BUG] Test case `test_node_default_disk_labeled` failed [7385](https://github.com/longhorn/longhorn/issues/7385) - @derekbit @roger-ryao - [BUG] Helm2 install error: 'lookup' function not defined in validate-psp-install.yaml [6318](https://github.com/longhorn/longhorn/issues/6318) - @innobead @roger-ryao - [BUG] Client in go-spdk-helper is stuck after encountering IO timeout [7395](https://github.com/longhorn/longhorn/issues/7395) - @derekbit @chriscchien - [BUG] DataEngineV2 Unable to attach a PV to a pod in the newer kernel [7190](https://github.com/longhorn/longhorn/issues/7190) - @yangchiu @derekbit - [BUG] orphaned pod pod_id found, but error not a directory occurred when trying to remove the volumes dir [3207](https://github.com/longhorn/longhorn/issues/3207) - @weizhe0422 @roger-ryao - [BUG] Download backing image failed with HTTP 502 error if Storage Network configured [7236](https://github.com/longhorn/longhorn/issues/7236) - @ChanYiLin @roger-ryao - [BUG] During volume live engine upgrade, delete replica with old engine image will make volume degraded forever [7012](https://github.com/longhorn/longhorn/issues/7012) - @PhanLe1010 @chriscchien - [BUG] A race after a node reboot leads to I/O errors with migratable volumes [6961](https://github.com/longhorn/longhorn/issues/6961) - @yangchiu @ejweber - [BUG] Metric totalVolumeSize and totalVolumeActualSize incorrect due to v2 volume counts [7380](https://github.com/longhorn/longhorn/issues/7380) - @c3y1huang @chriscchien - [BUG] Longhorn-manager does not deploy CSI driver when integrated with linkerd service mesh [3809](https://github.com/longhorn/longhorn/issues/3809) - @mantissahz @chriscchien - [BUG] Test case `test_node_eviction` failed [7210](https://github.com/longhorn/longhorn/issues/7210) - @ejweber @roger-ryao - [BUG] Cannot add block-type disk to node resource due to timeout error [7253](https://github.com/longhorn/longhorn/issues/7253) - @yangchiu @shuo-wu - [BUG] multiple "for-cloning-volume" snapshots created after cloning volume [5835](https://github.com/longhorn/longhorn/issues/5835) - @PhanLe1010 @chriscchien - [BUG] Volume has 2 active engines at the same time that blocks the volume controller reconciliation loop [4827](https://github.com/longhorn/longhorn/issues/4827) - @PhanLe1010 @chriscchien @roger-ryao - [BUG] Volume UI displays only the last backup when using the recurring job [2997](https://github.com/longhorn/longhorn/issues/2997) - @mantissahz @chriscchien @roger-ryao - [BUG] Volume gets stuck in an unknown state forever if created in an engine not fully deployed environment [6131](https://github.com/longhorn/longhorn/issues/6131) - @yangchiu @PhanLe1010 - [BUG] Continuously auto-balancing replicas when zone does not have enough space [6671](https://github.com/longhorn/longhorn/issues/6671) - @yangchiu @c3y1huang @roger-ryao - [BUG] `backing-image-manager-` hostPath selection exception [7062](https://github.com/longhorn/longhorn/issues/7062) - @ChanYiLin @chriscchien - [BUG] GET error for volume attachment on node reboot [4188](https://github.com/longhorn/longhorn/issues/4188) - @PhanLe1010 - [BUG] Errors found by static checker in volume controller [7009](https://github.com/longhorn/longhorn/issues/7009) - @m-ildefons - [BUG] Enabling replica-auto-balance tries to replicate to disabled nodes causing lots of errors in the logs and in the UI [6508](https://github.com/longhorn/longhorn/issues/6508) - @c3y1huang @chriscchien - [BUG] Confusing logging when trying to attach a new volume with no scheduled replicas [7244](https://github.com/longhorn/longhorn/issues/7244) - @ejweber @chriscchien - [BUG] `allow-collecting-longhorn-usage-metrics` setting is missing from chart settings [7050](https://github.com/longhorn/longhorn/issues/7050) - @ChanYiLin @yardenshoham @roger-ryao - [BUG] Longhorn storage network is incompatible with Multus version above v4.0.0 [6953](https://github.com/longhorn/longhorn/issues/6953) - @c3y1huang @chriscchien - [BUG] The archived docs page is broken [7222](https://github.com/longhorn/longhorn/issues/7222) - @innobead - [IMPROVEMENT] Optimize the resource cache to prevent high memory usage in longhorn-manager [6954](https://github.com/longhorn/longhorn/issues/6954) - @derekbit @nitendra-suse - [DOC] longhorn-csi-plugin stuck in CrashLoopBackOff after system crash (SELinux related) [5348](https://github.com/longhorn/longhorn/issues/5348) - @ejweber - [BUG] Cannot detach the restored volume when there is a node goes down during restoring [2103](https://github.com/longhorn/longhorn/issues/2103) - @ejweber @chriscchien - [BUG] Failing to mount encrypted volumes [7033](https://github.com/longhorn/longhorn/issues/7033) - @mantissahz @chriscchien - [BUG] The instance manager with state unknown will be cleaned up in the split-brain case [6479](https://github.com/longhorn/longhorn/issues/6479) - @shuo-wu @chriscchien - [BUG] Orphan snapshot attachment tickets prevent volume from detaching [6652](https://github.com/longhorn/longhorn/issues/6652) - @ejweber @chriscchien - [BUG] Test case `test_system_backup_and_restore` failed [7143](https://github.com/longhorn/longhorn/issues/7143) - @ChanYiLin @roger-ryao - [BUG] missing description in support-bundle metadata.yaml [6997](https://github.com/longhorn/longhorn/issues/6997) - @c3y1huang @roger-ryao - [BUG] Cannot mount XFS PV [7140](https://github.com/longhorn/longhorn/issues/7140) - @PhanLe1010 @roger-ryao - [BUG] Volume encryption doesn't work on Amazon Linux 2 [5944](https://github.com/longhorn/longhorn/issues/5944) - @derekbit @chriscchien - [BUG] Test case `test_csi_minimal_volume_size` failed [7170](https://github.com/longhorn/longhorn/issues/7170) - @roger-ryao - [BUG] Deleting a PVC bound to a CSI PV, will delete associated volume and the CSI PV in result. [7172](https://github.com/longhorn/longhorn/issues/7172) - - [BUG] Relax S3 client retry intervals, for throttled requests [2810](https://github.com/longhorn/longhorn/issues/2810) - @mantissahz @chriscchien - [BUG] supportbundle/kubelet.log empty in k3s environment [7121](https://github.com/longhorn/longhorn/issues/7121) - @c3y1huang @chriscchien - [BUG] Failing to mount encrypted volumes v1.5.2 [7045](https://github.com/longhorn/longhorn/issues/7045) - @derekbit @nitendra-suse - [BUG] Invalid volume name containing less-than sign [7092](https://github.com/longhorn/longhorn/issues/7092) - - [BUG] Somehow the Rebuilding field inside volume.meta is set to true when one replica only, causing the volume into attaching/detaching loop [6626](https://github.com/longhorn/longhorn/issues/6626) - @c3y1huang @nitendra-suse - [BUG] [longhorn-engine] [s390x] intermittent fail pipeline on build step [6975](https://github.com/longhorn/longhorn/issues/6975) - @Anarkis - [BUG] Longhorn Read-Only setting can be modified [5989](https://github.com/longhorn/longhorn/issues/5989) - @mantissahz @roger-ryao - [BUG] UI: All components handle window resizing events incorrectly [7036](https://github.com/longhorn/longhorn/issues/7036) - @votdev - [BUG] UI: The action menu handler should stop event propagation [7032](https://github.com/longhorn/longhorn/issues/7032) - @votdev - [BUG] longhorn manager isn't annotated with iam.amazonaws.com/role [6947](https://github.com/longhorn/longhorn/issues/6947) - @mantissahz @chriscchien - [BUG] invalid memory address or nil pointer dereference in BackupVolumeController [6998](https://github.com/longhorn/longhorn/issues/6998) - @mantissahz @roger-ryao - [BUG] Longhorn manager pods in 1.5.1 consuming 20GB+ RAM and 3-4 vCPUs [6866](https://github.com/longhorn/longhorn/issues/6866) - @derekbit @shuo-wu - [BUG] MountVolume.MountDevice failed for volume Output: mount.nfs: Protocol not supported [6887](https://github.com/longhorn/longhorn/issues/6887) - @derekbit - [BUG] High CPU usage on one node. [6578](https://github.com/longhorn/longhorn/issues/6578) - @derekbit @chriscchien - [BUG] Set a invalid backup target when backup in progress will cause backup never finish [6491](https://github.com/longhorn/longhorn/issues/6491) - @ChanYiLin @chriscchien - [BUG] duplicate MIME type "text/html" in `/var/config/nginx/nginx.conf` [7002](https://github.com/longhorn/longhorn/issues/7002) - @votdev - [BUG] After crashed engine process, volume stuck in `Unknown` state [6699](https://github.com/longhorn/longhorn/issues/6699) - @ChanYiLin @nitendra-suse - [BUG] Longhorn Instance Manager Memory leak [6481](https://github.com/longhorn/longhorn/issues/6481) - @james-munson @chriscchien - [BUG] Two active engine when volume migrating [6642](https://github.com/longhorn/longhorn/issues/6642) - @PhanLe1010 @chriscchien - [BUG] Button "Take Snapshot" and "Create Backup" grayed out. [6841](https://github.com/longhorn/longhorn/issues/6841) - @votdev - [BUG] Environment Check Script Fails To Perform All Checks [5653](https://github.com/longhorn/longhorn/issues/5653) - @PhanLe1010 @roger-ryao - [BUG] Volumes failing to mount because of engine upgradedReplicaAddressMap reference [6762](https://github.com/longhorn/longhorn/issues/6762) - @PhanLe1010 @chriscchien - [BUG] Unable to add a block-type disk with a new name [6849](https://github.com/longhorn/longhorn/issues/6849) - @derekbit @chriscchien - [BUG] IO error occurs when detaching RWX volume [6829](https://github.com/longhorn/longhorn/issues/6829) - @derekbit @chriscchien - [BUG] DR volume failed when synchronizing the incremental backup [6750](https://github.com/longhorn/longhorn/issues/6750) - @mantissahz @chriscchien - [BUG] Salvage failing in attaching and detaching loop, another pod is attached with health unknown [6662](https://github.com/longhorn/longhorn/issues/6662) - @james-munson - [BUG] 1.5.0: AttachVolume.Attach failed for volume, the volume is currently attached to different node [6287](https://github.com/longhorn/longhorn/issues/6287) - @yangchiu @derekbit - [BUG] Helm installation with privateRegistry.registryUrl set doesn't work [3057](https://github.com/longhorn/longhorn/issues/3057) - @PhanLe1010 @chriscchien - [BUG] cifs backup mount paths with dollar sign are not allowed [6660](https://github.com/longhorn/longhorn/issues/6660) - @derekbit @roger-ryao - [BUG] Failed Statefulset Pod Creation with RWX Workload on Longhorn v1.3.3 and SLES 15 SP5 [6494](https://github.com/longhorn/longhorn/issues/6494) - @ejweber @roger-ryao - [BUG] Failure to update backup status leads to infinite reconciliation [6358](https://github.com/longhorn/longhorn/issues/6358) - @ejweber @chriscchien - [BUG] longhorn installation randomly failed on sles 15-sp5 due to longhorn manager CrashLoopBackOff [6504](https://github.com/longhorn/longhorn/issues/6504) - @ejweber @chriscchien - [BUG] Can't delete volumesnapshot if backup target not set [4979](https://github.com/longhorn/longhorn/issues/4979) - @ejweber @chriscchien - [BUG] Share manager pod will stay in IO error when the volume becomes read only [5961](https://github.com/longhorn/longhorn/issues/5961) - @ChanYiLin @roger-ryao - [BUG] SettingNameSnapshotDataIntegrityCronJob should be sent as boolean value [6410](https://github.com/longhorn/longhorn/issues/6410) - @c3y1huang @roger-ryao - [BUG] Permission denied when starting longhorn-ui container [6430](https://github.com/longhorn/longhorn/issues/6430) - @mantissahz @chriscchien - [BUG] Longhorn manager crashed during backing image 100gb volume export [5209](https://github.com/longhorn/longhorn/issues/5209) - @ChanYiLin @chriscchien - [BUG] Removed IM CPU request settings still exists and new IM CPU request missed from chart settings [6465](https://github.com/longhorn/longhorn/issues/6465) - @c3y1huang @chriscchien - [BUG] Error during backup process will be removed quickly without user knowing [1249](https://github.com/longhorn/longhorn/issues/1249) - @mantissahz @chriscchien - [BUG] PV using v2 engine cannot attach [6441](https://github.com/longhorn/longhorn/issues/6441) - @derekbit @chriscchien @nitendra-suse - [BUG] Backup Job returns "Completed" despite running into errors [4255](https://github.com/longhorn/longhorn/issues/4255) - @mantissahz @chriscchien - [BUG] 1.5.0 Upgrade: Longhorn conversion webhook server fails [6259](https://github.com/longhorn/longhorn/issues/6259) - @derekbit @roger-ryao - [BUG] Webhook is never called for BackingImageManager [6328](https://github.com/longhorn/longhorn/issues/6328) - @ejweber @chriscchien - [BUG] Error message not getting cleaned up on switching the backupstore [2944](https://github.com/longhorn/longhorn/issues/2944) - @mantissahz - [BUG] Unable to list backup from a local backupstore in RKE2 CIS-1.23 environment [6342](https://github.com/longhorn/longhorn/issues/6342) - @mantissahz - [BUG] test case test_inc_restoration_with_multiple_rebuild_and_expansion randomly failed [5496](https://github.com/longhorn/longhorn/issues/5496) - @mantissahz - [BUG] disk monitor cannot recognize disks if disk paths are somehow changed after reboot [6125](https://github.com/longhorn/longhorn/issues/6125) - @yangchiu @derekbit - [BUG] Can not delete type=`bi` VolumeSnapshot if related backing image not exist [6266](https://github.com/longhorn/longhorn/issues/6266) - @ChanYiLin @chriscchien - [BUG] Race leaves snapshot CRs that cannot be deleted [6298](https://github.com/longhorn/longhorn/issues/6298) - @yangchiu @PhanLe1010 @ejweber - [BUG] test case test_setting_priority_class failed in master and v1.5.x [6319](https://github.com/longhorn/longhorn/issues/6319) - @derekbit @chriscchien - [BUG] Upgrade to 1.5.0 failed: validator.longhorn.io denied the request if having orphan resources [6246](https://github.com/longhorn/longhorn/issues/6246) - @derekbit @roger-ryao - [BUG] Longhorn Manager Pods CrashLoop after upgrade from 1.4.0 to 1.5.0 while backing up volumes [6264](https://github.com/longhorn/longhorn/issues/6264) - @ChanYiLin @roger-ryao - [BUG] Unable to receive support bundle from UI when it's large (400MB+) [6256](https://github.com/longhorn/longhorn/issues/6256) - @c3y1huang @chriscchien - [BUG] Live upgrade stuck if the same volume name backup exists in the backup store [3403](https://github.com/longhorn/longhorn/issues/3403) - @ChanYiLin @chriscchien - [BUG] Instance manager may not update instance status for a minute after starting [5809](https://github.com/longhorn/longhorn/issues/5809) - @ejweber @chriscchien ### Performance - [FEATURE] Increase read bandwidth of v2 volume from all downstream replicas [5759](https://github.com/longhorn/longhorn/issues/5759) - @derekbit @chriscchien - [TASK] Add 1.5 performance benchmark to performance benchmark WIKI page [6203](https://github.com/longhorn/longhorn/issues/6203) - @derekbit ### Miscellaneous - [TASK] Bump the versions of dependent libs or components [7001](https://github.com/longhorn/longhorn/issues/7001) - @c3y1huang @chriscchien - [DOC] Make `Troubleshooting` section as an individual chapter [7706](https://github.com/longhorn/longhorn/issues/7706) - @derekbit - [TASK] Update the descriptions of setting variables in chart after doc review [7667](https://github.com/longhorn/longhorn/issues/7667) - @ChanYiLin - [REFACTOR] Remove unnecessary Kubernetes version check in chart manifests [7601](https://github.com/longhorn/longhorn/issues/7601) - @c3y1huang @roger-ryao - [TASK] Bump up the minimum supported Kubernetes version [7224](https://github.com/longhorn/longhorn/issues/7224) - @c3y1huang @roger-ryao - [TASK] Update CLIAPIVersion in longhorn-manager [7588](https://github.com/longhorn/longhorn/issues/7588) - @FrankYang0529 @roger-ryao - [TASK] Security vulnerabilities in docker images [7523](https://github.com/longhorn/longhorn/issues/7523) - @c3y1huang @roger-ryao - [TASK][UI] v2 volume does not support engine image upgrade [7445](https://github.com/longhorn/longhorn/issues/7445) - @chriscchien @scures @roger-ryao - [DOC] Add missing descriptions for Helm [7485](https://github.com/longhorn/longhorn/issues/7485) - @mantissahz - [TASK] Update protoc to v24.3 [6666](https://github.com/longhorn/longhorn/issues/6666) - @FrankYang0529 - [FEATURE] Enable resource profiling for IM [6377](https://github.com/longhorn/longhorn/issues/6377) - @derekbit @roger-ryao - [TASK] Synchronize version of CSI components in longhorn/longhorn and longhorn/longhorn-manager [7377](https://github.com/longhorn/longhorn/issues/7377) - @c3y1huang @roger-ryao - [TASK] Upgrade csi-snapshotter to mitigate rapid retry bug [6506](https://github.com/longhorn/longhorn/issues/6506) - @ejweber - [TASK] Remove engine image dependency of v2 volumes [7157](https://github.com/longhorn/longhorn/issues/7157) - @derekbit - [DOC] Fix erroneous value for default StorageMinimalAvailablePercentage setting. [7342](https://github.com/longhorn/longhorn/issues/7342) - @james-munson - [DOC] FS Trim for RWX is supported, but docs are out of date. [6733](https://github.com/longhorn/longhorn/issues/6733) - @james-munson - [REFACTOR] Abstract the disk/lvol file operations in backupstore [6576](https://github.com/longhorn/longhorn/issues/6576) - @derekbit @chriscchien - [TASK] Implement xattr get and set operations on SPDK logical volumes (lvol) [6604](https://github.com/longhorn/longhorn/issues/6604) - @derekbit - [DOC] Stress using object store as best practice for backups. [6773](https://github.com/longhorn/longhorn/issues/6773) - @james-munson - [DOC] Run fsck.ext4 on newer Longhorn volume from older Linux distro [6859](https://github.com/longhorn/longhorn/issues/6859) - @ejweber @roger-ryao - [TASK] Move common functions for backup to backupstore lib [6514](https://github.com/longhorn/longhorn/issues/6514) - @derekbit - [TASK] Investigate SELinux enabled with Longhorn [6074](https://github.com/longhorn/longhorn/issues/6074) - @yangchiu @ejweber - [IMPROVEMENT] List of Longhorn Helm Chart Flags [5455](https://github.com/longhorn/longhorn/issues/5455) - @ChanYiLin - [REFACTOR] UI: Disable `Delete` menu for default engine image [7029](https://github.com/longhorn/longhorn/issues/7029) - @votdev - [TASK][UI] Replace `spec.engineImage` field in volume, engine and replica CRDs with `spec.image` [6685](https://github.com/longhorn/longhorn/issues/6685) - @votdev - [EPIC] Side effects of increasing resync period in informer's event handlers [3629](https://github.com/longhorn/longhorn/issues/3629) - @PhanLe1010 - [TASK] The development branch should reference to the head images in longhorn-image.txt [6737](https://github.com/longhorn/longhorn/issues/6737) - @c3y1huang @chriscchien - [TASK] Create a CIFS backup store example in longhorn repo [6530](https://github.com/longhorn/longhorn/issues/6530) - @chriscchien - [DOC] Explanation of storage class parameters [4776](https://github.com/longhorn/longhorn/issues/4776) - @james-munson @roger-ryao - [DOC] Create a KB for high space consumption issue guideline [6592](https://github.com/longhorn/longhorn/issues/6592) - @shuo-wu - [DOC] Create a KB for incorrect replica expansion [6391](https://github.com/longhorn/longhorn/issues/6391) - @ejweber - [DOC] `deploy/longhorn.yaml` out of date - causes all longhorn-manager instances to crash-loop [6428](https://github.com/longhorn/longhorn/issues/6428) - @c3y1huang - [REFACTORING] Move adding finalizer of resources to mutation webhooks as volume/engine/replica [4872](https://github.com/longhorn/longhorn/issues/4872) - @ejweber @chriscchien - [TASK] Update or remove out-of-date cleanup script [6316](https://github.com/longhorn/longhorn/issues/6316) - @james-munson - [DOC] v1.5.0 additional outgoing firewall ports need to be opened 9501 9502 9503 [6317](https://github.com/longhorn/longhorn/issues/6317) - @ChanYiLin @chriscchien - [TASK] Check and update the networking doc & example YAMLs [5651](https://github.com/longhorn/longhorn/issues/5651) - @yangchiu @shuo-wu ## Contributors - @Anarkis - @ArthurVardevanyan - @ChanYiLin - @DamiaSan - @FrankYang0529 - @PhanLe1010 - @Vicente-Cheng - @antoninferrand - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @jillian-maroket - @m-ildefons - @mantissahz - @nitendra-suse - @roger-ryao - @scures - @sfackler - @shuo-wu - @smallteeths - @votdev - @weizhe0422 - @yangchiu - @yardenshoham ================================================ FILE: CHANGELOG/CHANGELOG-1.6.1.md ================================================ ## Longhorn v1.6.1 Release Notes Longhorn 1.6.1 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > **Note:** > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.6.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.6.1/deploy/install/) in the Longhorn documentation. ## Upgrade **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.5.x or v1.6.x to v1.6.1.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.6.1/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.6.1/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.6.1][IMPROVEMENT] Add dmsetup and dmcrypt utilities check in environment check script [8234](https://github.com/longhorn/longhorn/issues/8234) - @derekbit @chriscchien - [BACKPORT][v1.6.1][IMPROVEMENT] Upgrade support bundle kit version to v0.0.36 [8162](https://github.com/longhorn/longhorn/issues/8162) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.1][IMPROVEMENT] Cannot read/write to block volume when the container is run as non-root [8122](https://github.com/longhorn/longhorn/issues/8122) - @PhanLe1010 @chriscchien - [BACKPORT][v1.6.1][IMPROVEMENT] Improve environment_check script for NFS protocol bug and the host system self diagnosis [7971](https://github.com/longhorn/longhorn/issues/7971) - @james-munson @roger-ryao - [BACKPORT][v1.6.1][IMPROVEMENT] Use HEAD instead of a GET to fetch the `Content-Length` of an resource via URL [7973](https://github.com/longhorn/longhorn/issues/7973) - @votdev @roger-ryao - [BACKPORT][v1.6.1][IMPROVEMENT] Remove startup probe of CSI driver after liveness probe conn fix ready [7886](https://github.com/longhorn/longhorn/issues/7886) - @ejweber @roger-ryao - [BACKPORT][v1.6.1][IMPROVEMENT] Change support-bundle-manager image pull policy to PullIfNotPresent [8000](https://github.com/longhorn/longhorn/issues/8000) - @ChanYiLin @roger-ryao ### Bug - [BUG][v1.6.1-rc3] test_backup_lock_creation_during_deletion failed with *.lck type 1 acquisition [8269](https://github.com/longhorn/longhorn/issues/8269) - @ChanYiLin @chriscchien - [BACKPORT][v1.6.1][BUG] Replica rebuild failed [8258](https://github.com/longhorn/longhorn/issues/8258) - @shuo-wu @roger-ryao - [BACKPORT][v1.6.1][BUG] potential risk to unmap a negative number [8237](https://github.com/longhorn/longhorn/issues/8237) - @Vicente-Cheng @roger-ryao - [BACKPORT][v1.6.1][BUG] Can't use longhorn with Generic ephemeral volumes [8213](https://github.com/longhorn/longhorn/issues/8213) - @ejweber @chriscchien - [BUG] [v1.6.1-rc1] v2 volume replica offline rebuilding fail [8187](https://github.com/longhorn/longhorn/issues/8187) - @shuo-wu @chriscchien - [BACKPORT][v1.6.1][BUG] Longhorn api-server PUT request rate [8152](https://github.com/longhorn/longhorn/issues/8152) - @ejweber @roger-ryao - [BACKPORT][v1.6.1][BUG] Use config map to update `default-replica-count` won't apply to `default-replica-count.definition.default` if the value equal to current `default-replica-count.value` [8134](https://github.com/longhorn/longhorn/issues/8134) - @james-munson @roger-ryao - [BACKPORT][v1.6.1][BUG] Failed to restore a backup to file by the scripts/restore-backup-to-file.sh with a CIFS backup target. [8128](https://github.com/longhorn/longhorn/issues/8128) - @mantissahz @roger-ryao - [BACKPORT][v1.6.1][BUG] Exporting data from the existing replicas has issues in 1.6.0 [8096](https://github.com/longhorn/longhorn/issues/8096) - @ChanYiLin @chriscchien - [BACKPORT][v1.6.1][BUG] longhorn manager pod fails to start in container-based K3s [7847](https://github.com/longhorn/longhorn/issues/7847) - @ChanYiLin @khushboo-rancher @chriscchien - [BACKPORT][v1.6.1][BUG] no Pending workload pods for volume xxx to be mounted [8081](https://github.com/longhorn/longhorn/issues/8081) - @c3y1huang @chriscchien - [BACKPORT][v1.6.1][BUG] Deadlock is possible in v1.6.0 instance manager [7920](https://github.com/longhorn/longhorn/issues/7920) - @roger-ryao - [BACKPORT][v1.6.1][BUG][1.6.0] ENGINE v2 : disk /dev/xxxx is already used by AIO bdev disk-x [8133](https://github.com/longhorn/longhorn/issues/8133) - @derekbit @chriscchien - [BACKPORT][v1.6.1][BUG] A replica may be incorrectly scheduled to a node with an existing failed replica [8044](https://github.com/longhorn/longhorn/issues/8044) - @ejweber @chriscchien - [BACKPORT][v1.6.1][BUG] Deadlock between volume migration and upgrade after Longhorn upgrade [7870](https://github.com/longhorn/longhorn/issues/7870) - @ejweber @roger-ryao - [BACKPORT][v1.6.1][BUG] Volume cannot attach because of the leftover non-empty volume.status.PendingNodeID after upgrading Longhorn [7995](https://github.com/longhorn/longhorn/issues/7995) - @james-munson @chriscchien - [BACKPORT][v1.6.1][BUG] Longhorn may keep corrupted salvaged replicas and discard good ones [7885](https://github.com/longhorn/longhorn/issues/7885) - @ejweber @roger-ryao - [BACKPORT][v1.6.1][BUG] LH manager reboots due to the webhook is not ready [8037](https://github.com/longhorn/longhorn/issues/8037) - @ChanYiLin @chriscchien - [BACKPORT][v1.6.1][BUG] BackingImage does not download URL correctly in some situation [7987](https://github.com/longhorn/longhorn/issues/7987) - @votdev @yangchiu - [BACKPORT][v1.6.1][BUG] Executing fstrim while rebuilding causes IO errors [7868](https://github.com/longhorn/longhorn/issues/7868) - @yangchiu @ejweber - [BACKPORT][v1.6.1][BUG] The feature of auto remount read only volume not work on a single node cluster. [7846](https://github.com/longhorn/longhorn/issues/7846) - @yangchiu @ChanYiLin - [BACKPORT][v1.6.1][BUG] Missed NodeStageVolume after reboot leads to CreateContainerError [8012](https://github.com/longhorn/longhorn/issues/8012) - @ejweber @chriscchien - [BACKPORT][v1.6.1][BUG] persistence.removeSnapshotsDuringFilesystemTrim Helm variable is unreferenced [7952](https://github.com/longhorn/longhorn/issues/7952) - @ejweber @chriscchien - [BACKPORT][v1.6.1][BUG][v1.5.4-rc4] Test case test_backuptarget_available_during_engine_image_not_ready failed to wait for backup target available [8055](https://github.com/longhorn/longhorn/issues/8055) - @c3y1huang @chriscchien - [BACKPORT][v1.6.1][BUG] Add Snapshot Maximum Count to the Settings [7979](https://github.com/longhorn/longhorn/issues/7979) - @FrankYang0529 @roger-ryao - [BACKPORT][v1.6.1][BUG] Volumes stuck upgrading after 1.5.3 -> 1.6.0 upgrade. [7899](https://github.com/longhorn/longhorn/issues/7899) - @ejweber @roger-ryao - [BACKPORT][v1.6.1][BUG] The activated DR volume do not contain the latest data. [7946](https://github.com/longhorn/longhorn/issues/7946) - @shuo-wu @roger-ryao - [BACKPORT][v1.6.1][BUG][v1.5.x] Recurring job fails to create backup when volume detached [8013](https://github.com/longhorn/longhorn/issues/8013) - @yangchiu @mantissahz @PhanLe1010 @c3y1huang - [BACKPORT][v1.6.1][BUG] Create backup failed: failed lock lock-*.lck type 1 acquisition [7875](https://github.com/longhorn/longhorn/issues/7875) - @yangchiu @ChanYiLin @chriscchien - [BUG] Fix errors in questions.yaml [6392](https://github.com/longhorn/longhorn/issues/6392) - @james-munson @chriscchien ### Misc - [BACKPORT][v1.6.1][REFACTOR] move mount point check function to common lib [8124](https://github.com/longhorn/longhorn/issues/8124) - @ChanYiLin @chriscchien - [TASK] Fix updating patch digest dependencies (v1.6.x) [8107](https://github.com/longhorn/longhorn/issues/8107) - @mantissahz ## Contributors - @ChanYiLin - @FrankYang0529 - @PhanLe1010 - @Vicente-Cheng - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @khushboo-rancher - @mantissahz - @roger-ryao - @shuo-wu - @votdev - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.6.2.md ================================================ ## Longhorn v1.6.2 Release Notes Longhorn 1.6.2 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.6.2.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.6.2/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.5.x or v1.6.x (< v1.6.2) to v1.6.2.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.6.2/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.6.2/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.6.2][IMPROVEMENT] Saving Settings page changes [8600](https://github.com/longhorn/longhorn/issues/8600) - @a110605 @roger-ryao - [BACKPORT][v1.6.2][IMPROVEMENT] Expose virtual size of qcow2 backing images [8322](https://github.com/longhorn/longhorn/issues/8322) - @chriscchien - [BACKPORT][v1.6.2][IMPROVEMENT] Prevent unnecessary updates of instanceManager status [8421](https://github.com/longhorn/longhorn/issues/8421) - @yangchiu @derekbit - [BACKPORT][v1.6.2][UI][IMPROVEMENT] Allow users to request backup volume update [8539](https://github.com/longhorn/longhorn/issues/8539) - @a110605 @chriscchien - [BACKPORT][v1.6.2][IMPROVEMENT] Allow users to request backup volume update [8154](https://github.com/longhorn/longhorn/issues/8154) - @yangchiu @mantissahz - [BACKPORT][v1.6.2][IMPROVEMENT] Investigate performance bottleneck in v1 data path [8511](https://github.com/longhorn/longhorn/issues/8511) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.6.2]Mirror the `quay.io/openshift/origin-oauth-proxy` image to Longhorn repo similar to what we are doing for CSI sidecar images [8334](https://github.com/longhorn/longhorn/issues/8334) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.6.2][IMPROVEMENT] Avoid misleading log messages in longhorn manager while syncing danger zone settings [8383](https://github.com/longhorn/longhorn/issues/8383) - @yangchiu @mantissahz - [BACKPORT][v1.6.2][IMPROVEMENT] Do not terminate nfs-ganesha in share-manager pod after failing to access recovery backend [8346](https://github.com/longhorn/longhorn/issues/8346) - @derekbit @chriscchien - [BACKPORT][v1.6.2][IMPROVEMENT] Improve environment_check script for NFS protocol bug and the host system self diagnosis [8277](https://github.com/longhorn/longhorn/issues/8277) - @james-munson @chriscchien ### Bug - [BACKPORT][v1.6.2][BUG] Longhorn upgrade from 1.4.4 to 1.5.5 failing [8607](https://github.com/longhorn/longhorn/issues/8607) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.6.2][BUG] BackupTarget conditions don't reflect connection errors in v1.6.0 [8223](https://github.com/longhorn/longhorn/issues/8223) - @ejweber @chriscchien - [BACKPORT][v1.6.2][BUG] share-manager-pvc appears to be leaking memory [8426](https://github.com/longhorn/longhorn/issues/8426) - @derekbit @roger-ryao - [BACKPORT][v1.6.2][BUG] Secret for backup not found [8512](https://github.com/longhorn/longhorn/issues/8512) - @yangchiu @mantissahz - [BUG][v1.6.2-rc1] Workload pod got stuck in ContainerStatusUnknown after node shutdown and reboot [8550](https://github.com/longhorn/longhorn/issues/8550) - @c3y1huang - [BACKPORT][v1.6.2][BUG] Valid backup secret produces error message: "there is space or new line in AWS_CERT" [8477](https://github.com/longhorn/longhorn/issues/8477) - @yangchiu @mantissahz - [BACKPORT][v1.6.2][BUG] Backup marked as "completed" cannot be restored, gzip: invalid header [8377](https://github.com/longhorn/longhorn/issues/8377) - @derekbit @roger-ryao - [BACKPORT][v1.6.2][BUG] Lost connection to unix:///csi/csi.sock [8493](https://github.com/longhorn/longhorn/issues/8493) - @ejweber @roger-ryao - [BACKPORT][v1.6.2][BUG] Longhorn Helm uninstall times out. [8409](https://github.com/longhorn/longhorn/issues/8409) - @ChanYiLin @roger-ryao - [BACKPORT][v1.6.2][BUG] Disable tls 1.0 and 1.1 on webhook service [8388](https://github.com/longhorn/longhorn/issues/8388) - @ChanYiLin @roger-ryao - [BACKPORT][v1.6.2][BUG] longhorn-manager build failed [8410](https://github.com/longhorn/longhorn/issues/8410) - @yangchiu @mantissahz - [BACKPORT][v1.6.2][BUG] RWX volume is hang on Photon OS [8279](https://github.com/longhorn/longhorn/issues/8279) - @yangchiu @PhanLe1010 ### Misc - [BACKPORT][v1.6.2]DOCS - Incorrect documentation on pre-upgrade checker configuration [8342](https://github.com/longhorn/longhorn/issues/8342) - @yangchiu ## Contributors - @ChanYiLin - @PhanLe1010 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @forbesguthrie - @innobead - @james-munson - @jillian-maroket - @mantissahz - @rebeccazzzz - @roger-ryao - @shuo-wu - @yangchiu ================================================ FILE: CHANGELOG/CHANGELOG-1.6.3.md ================================================ ## Longhorn v1.6.3 Release Notes Longhorn 1.6.3 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.6.3.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.6.3/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.5.x or v1.6.x (< v1.6.3) to v1.6.3.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.6.3/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.6.3/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Features - [BACKPORT][v1.6.3][FEATURE] Add additional monitoring settings to ServiceMonitor resource. [8984](https://github.com/longhorn/longhorn/issues/8984) - @ejweber @chriscchien ### Improvement - [BACKPORT][v1.6.3][IMPROVEMENT] Fix contradicting node status events [9327](https://github.com/longhorn/longhorn/issues/9327) - @ejweber @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] Always update the built-in installed system packages when building component images [8722](https://github.com/longhorn/longhorn/issues/8722) - @yangchiu @c3y1huang - [BACKPORT][v1.6.3][IMPROVEMENT] Update sizes in Engine and Volume resources less frequently [8684](https://github.com/longhorn/longhorn/issues/8684) - @ejweber @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] Longhor Manager Flood with "Failed to get engine proxy of ... cannot get client for engine" Message [8729](https://github.com/longhorn/longhorn/issues/8729) - @derekbit @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] Restore Latest Backup should be applied with BackingImage name value [8671](https://github.com/longhorn/longhorn/issues/8671) - @a110605 @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] Improve and simplify chart values.yaml [8636](https://github.com/longhorn/longhorn/issues/8636) - @ChanYiLin @chriscchien - [BACKPORT][v1.6.3][IMPROVEMENT] BackingImage UI improvement [8655](https://github.com/longhorn/longhorn/issues/8655) - @a110605 @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] Saving Settings page changes [8602](https://github.com/longhorn/longhorn/issues/8602) - @a110605 @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] The client-go rest client rate limit inside the csi sidecar component might be too small (csi-provisioner, csi-attacjer. csi-snappshotter, csi-attacher) [8726](https://github.com/longhorn/longhorn/issues/8726) - @PhanLe1010 - [BACKPORT][v1.6.3][IMPROVEMENT] Add setting to configure support bundle timeout for node bundle collection [8624](https://github.com/longhorn/longhorn/issues/8624) - @c3y1huang @chriscchien - [BACKPORT][v1.6.3][IMPROVEMENT] Problems mounting XFS volume clones / restored snapshots [8797](https://github.com/longhorn/longhorn/issues/8797) - @PhanLe1010 @chriscchien - [BACKPORT][v1.6.3][IMPROVEMENT] Cannot expand a volume created by Longhorn UI [8828](https://github.com/longhorn/longhorn/issues/8828) - @mantissahz - [BACKPORT][v1.6.3][IMPROVEMENT] environment_check.sh should check for the iscsi_tcp kernel module [8720](https://github.com/longhorn/longhorn/issues/8720) - @tserong @roger-ryao - [BACKPORT][v1.6.3][IMPROVEMENT] `toomanysnapshots` UI element not prominent enough to prevent runaway snapshots [8672](https://github.com/longhorn/longhorn/issues/8672) - @a110605 @roger-ryao ### Bug - [BACKPORT][v1.6.3][BUG] instance-manager is stuck at starting state [8678](https://github.com/longhorn/longhorn/issues/8678) - @derekbit - [BACKPORT][v1.6.3][BUG] kubectl drain node is blocked by unexpected orphan engine processes [9446](https://github.com/longhorn/longhorn/issues/9446) - @ejweber @chriscchien @roger-ryao - [BACKPORT][v1.6.3][BUG] Uninstallation will fail if invalid backuptarget is set. [8793](https://github.com/longhorn/longhorn/issues/8793) - @mantissahz @chriscchien - [BACKPORT][v1.6.3][BUG] Longhorn thinks node is unschedulable [9052](https://github.com/longhorn/longhorn/issues/9052) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.3][BUG] Can not revert V2 volume snapshot after upgrade from v1.6.2 to v1.7.0-dev [9066](https://github.com/longhorn/longhorn/issues/9066) - @chriscchien @DamiaSan - [BACKPORT][v1.6.3][BUG] Canceling expansion results in a volume expansion error [9469](https://github.com/longhorn/longhorn/issues/9469) - @derekbit - [BACKPORT][v1.6.3][BUG] Pod auto-deletion may cause thousands of logs [9020](https://github.com/longhorn/longhorn/issues/9020) - @ejweber @roger-ryao - [BACKPORT][v1.6.3][BUG] Engine Upgrade to 1.7.1 fails on volumes with strict-local data locality [9447](https://github.com/longhorn/longhorn/issues/9447) - @james-munson @chriscchien - [BACKPORT][v1.6.3][BUG] Fix longhorn-manager `TestCleanupRedundantInstanceManagers` [8670](https://github.com/longhorn/longhorn/issues/8670) - @derekbit @roger-ryao - [BUG] Security issues in longhorn 1.6.2 version images [9132](https://github.com/longhorn/longhorn/issues/9132) - @c3y1huang - [BACKPORT][v1.6.3][BUG] Longhorn keeps resetting my storageClass [9395](https://github.com/longhorn/longhorn/issues/9395) - @mantissahz @roger-ryao - [BUG] Regression in 1.6.x-head, significant increase in execution time [9439](https://github.com/longhorn/longhorn/issues/9439) - @ChanYiLin @roger-ryao - [BACKPORT][v1.6.3][BUG] System Backup Fails and DR Volume Enters Attach-Detach Loop When Volume Backup Policy is Set to `Always` [9339](https://github.com/longhorn/longhorn/issues/9339) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.3][BUG] `toomanysnapshots` UI message displays incorrect snapshot count [8700](https://github.com/longhorn/longhorn/issues/8700) - @ejweber - [BACKPORT][v1.6.3][BUG] Longhorn can no longer create XFS volumes smaller than 300 MiB [8560](https://github.com/longhorn/longhorn/issues/8560) - @ejweber @chriscchien - [BACKPORT][v1.6.3][BUG] test case `test_system_backup_and_restore_volume_with_backingimage` failed on sle-micro ARM64 [9227](https://github.com/longhorn/longhorn/issues/9227) - @ChanYiLin @roger-ryao - [BACKPORT][v1.6.3][BUG] Longhorn did not close and open encrypted volumes correctly when the service k3s-agent restarted for a while [9386](https://github.com/longhorn/longhorn/issues/9386) - @mantissahz @roger-ryao - [BUG] test case `test_recurring_job` the backup recurring job's retain is not working on `v1.6.x-head` for `amd64` [9454](https://github.com/longhorn/longhorn/issues/9454) - @mantissahz @chriscchien - [BUG][v1.6.x] Abnormal snapshot missing status field [9438](https://github.com/longhorn/longhorn/issues/9438) - @yangchiu @derekbit - [BACKPORT][v1.6.3][BUG] Instance manager missing required selector labels after manager crash [9472](https://github.com/longhorn/longhorn/issues/9472) - @c3y1huang @chriscchien - [BUG] test case `test_support_bundle_should_not_timeout` timeout on `v1.6.x-head` for `amd64` [9452](https://github.com/longhorn/longhorn/issues/9452) - @yangchiu @derekbit - [BACKPORT][v1.6.3][BUG] Replica Auto Balance options under General Setting and under Volume section should have similar case [8786](https://github.com/longhorn/longhorn/issues/8786) - @yangchiu @a110605 - [BUG][UI][v1.6.x] blank dropdown menu in update volume property modals [9465](https://github.com/longhorn/longhorn/issues/9465) - @a110605 - [BACKPORT][v1.6.3][BUG] Accidentally encountered a single replica volume backup stuck at progress 17% indefinitely after a node rebooted [9399](https://github.com/longhorn/longhorn/issues/9399) - @yangchiu @ChanYiLin - [BACKPORT][v1.6.3][BUG] Non-existing block device results in longhorn-manager to be in Crashloopbackoff state [9074](https://github.com/longhorn/longhorn/issues/9074) - @yangchiu @derekbit - [BACKPORT][v1.6.3][BUG] Volume failed to create healthy replica after data locality and replica count changed and got stuck in degraded state forever [8561](https://github.com/longhorn/longhorn/issues/8561) - @ejweber @chriscchien @roger-ryao - [BACKPORT][v1.6.3][BUG] [Backupstore] Need to close the reader after downloading files for the Azure backup store driver. [9283](https://github.com/longhorn/longhorn/issues/9283) - @yangchiu @mantissahz - [BACKPORT][v1.6.3][BUG] LH fails silently when node has attached volumes [9211](https://github.com/longhorn/longhorn/issues/9211) - @yangchiu @ejweber - [BACKPORT][v1.6.3][BUG] Volume stuck in degraded [9295](https://github.com/longhorn/longhorn/issues/9295) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.6.3][BUG] v1 volume replica rebuild fail after upgrade from v1.7.0 to v1.7.1-rc1 [9336](https://github.com/longhorn/longhorn/issues/9336) - @PhanLe1010 @chriscchien - [BACKPORT][v1.6.3][BUG] instance-manager pod for v2 volume is killed due to a failed liveness probe. [8808](https://github.com/longhorn/longhorn/issues/8808) - @derekbit @chriscchien - [BACKPORT][v1.6.3][BUG] Share manager controller reconciles tens of thousands of times [9088](https://github.com/longhorn/longhorn/issues/9088) - @ejweber @roger-ryao - [BACKPORT][v1.6.3][BUG] Scale replica snapsots warning [8851](https://github.com/longhorn/longhorn/issues/8851) - @ejweber - [BACKPORT][v1.6.3][BUG]filesystem trim RecurringJob times out (volumes where files are frequently created and deleted) [9048](https://github.com/longhorn/longhorn/issues/9048) - @c3y1huang @chriscchien - [BACKPORT][v1.6.3][BUG] Rebuilding Replica fails on larger volumes [8949](https://github.com/longhorn/longhorn/issues/8949) - - [BACKPORT][v1.6.3][BUG] Orphan longhorn-engine-manager and longhorn-replica-manager services [8858](https://github.com/longhorn/longhorn/issues/8858) - @PhanLe1010 @chriscchien - [BACKPORT][v1.6.3][BUG] When revision counter is disabled, the engine might choose a replica with a smaller head size to be the source of truth for auto-salvage [8661](https://github.com/longhorn/longhorn/issues/8661) - @PhanLe1010 ### Misc - [BACKPORT][v1.6.3][TASK] Update the best practice page to mention these broken kernels [8882](https://github.com/longhorn/longhorn/issues/8882) - @PhanLe1010 ## Contributors - @ChanYiLin - @DamiaSan - @PhanLe1010 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @innobead - @james-munson - @mantissahz - @roger-ryao - @tserong - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.6.4.md ================================================ ## Longhorn v1.6.4 Release Notes Longhorn 1.6.4 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.6.4.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.6.4/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.5.x or v1.6.x (< v1.6.4) to v1.6.4.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.6.4/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.6.4/deploy/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.6.4][IMPROVEMENT] Add support for JSON log format configuration in Longhorn components (UI, driver) [10080](https://github.com/longhorn/longhorn/issues/10080) - @chriscchien - [BACKPORT][v1.6.4][IMPROVEMENT] Logging the reason why the instance manager pod is going to be deleted. [9888](https://github.com/longhorn/longhorn/issues/9888) - @derekbit @chriscchien - [BACKPORT][v1.6.4][IMPROVEMENT] Check NFS versions in /etc/nfsmount.conf instead [9832](https://github.com/longhorn/longhorn/issues/9832) - @COLDTURNIP @yangchiu - [BACKPORT][v1.6.4][IMPROVEMENT] Prevent Volume Resize Stuck [9913](https://github.com/longhorn/longhorn/issues/9913) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.4][IMPROVEMENT] Reject strict-local + RWX volume creation [9931](https://github.com/longhorn/longhorn/issues/9931) - @COLDTURNIP @yangchiu - [BACKPORT][v1.6.4][IMPROVEMENT] Configure the log level of other system and user managed components via longhorn manager setting [9618](https://github.com/longhorn/longhorn/issues/9618) - @yangchiu @james-munson - [BACKPORT][v1.6.4][IMPROVEMENT] Change misleading error message to warning level [9918](https://github.com/longhorn/longhorn/issues/9918) - @yangchiu @derekbit - [BACKPORT][v1.6.4][IMPROVEMENT] Building longhorn-manager takes long time [9694](https://github.com/longhorn/longhorn/issues/9694) - @derekbit @chriscchien - [BACKPORT][v1.6.4][IMPROVEMENT] Remove mirrored openshift image from Longhorn [9599](https://github.com/longhorn/longhorn/issues/9599) - @derekbit @chriscchien ### Bug - [BUG][v1.6.x-head] Share manager pod kept restarting [10096](https://github.com/longhorn/longhorn/issues/10096) - @c3y1huang @chriscchien - [BACKPORT][v1.6.4][BUG] Webhook servers initialization blocks longhorn-manager from running [10067](https://github.com/longhorn/longhorn/issues/10067) - @c3y1huang - [BACKPORT][v1.6.4][BUG] Missing `fromBackup` Parameter in API Request When Restoring Multiple Files from Backup List [10065](https://github.com/longhorn/longhorn/issues/10065) - @a110605 @chriscchien - [BACKPORT][v1.6.4][BUG] Busrt ISCSI Connection Errors, and IM Pod Restarting to make LH Volume disconnection [9890](https://github.com/longhorn/longhorn/issues/9890) - @yangchiu @ChanYiLin @chriscchien - [BACKPORT][v1.6.4][BUG] Failed to inspect the backup backing image information if NFS backup target URL with options [9704](https://github.com/longhorn/longhorn/issues/9704) - @yangchiu @mantissahz @chriscchien - [BACKPORT][v1.6.4][BUG] Error notification appears on the volume backup details page [10070](https://github.com/longhorn/longhorn/issues/10070) - @a110605 @houhoucoop - [BACKPORT][v1.6.4][BUG][v1.8.x] Unable to add block disk after node deleted and added back [10041](https://github.com/longhorn/longhorn/issues/10041) - - [BACKPORT][v1.6.4][BUG] Detached Volume Stuck in Attached State During Node Eviction [9809](https://github.com/longhorn/longhorn/issues/9809) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.4][BUG] Backup progress should not add block failed to upload to successful count [9792](https://github.com/longhorn/longhorn/issues/9792) - @yangchiu @derekbit - [BACKPORT][v1.6.4][BUG] S3 Backup target reverts randomly to previous value [9589](https://github.com/longhorn/longhorn/issues/9589) - @c3y1huang - [BACKPORT][v1.6.4][BUG] Old backups are not cleaned up after timeout [9730](https://github.com/longhorn/longhorn/issues/9730) - @yangchiu @mantissahz - [BACKPORT][v1.6.4][BUG] Share manager is permanently stuck in stopping/error if we shutdown the node of share manager pod. This makes RWX PVC cannot attach to any new node [9855](https://github.com/longhorn/longhorn/issues/9855) - @yangchiu @PhanLe1010 - [BACKPORT][v1.6.4][BUG] Test case test_node_eviction_multiple_volume failed to reschedule replicas after volume detached [9867](https://github.com/longhorn/longhorn/issues/9867) - @yangchiu @c3y1huang - [BACKPORT][v1.6.4][BUG] DR volume fails to reattach and faulted after node stop and start during incremental restore [9802](https://github.com/longhorn/longhorn/issues/9802) - @c3y1huang @roger-ryao - [BACKPORT][v1.6.4][BUG] Fail to resize RWX PVC at filesystem resizing step [9737](https://github.com/longhorn/longhorn/issues/9737) - @james-munson - [BACKPORT][v1.6.4][BUG] Test case `Stopped replicas on deleted nodes should not be counted as healthy replicas when draining nodes` fails [9625](https://github.com/longhorn/longhorn/issues/9625) - @yangchiu @derekbit - [BACKPORT][v1.6.4][BUG] Pre-upgrade pod should event the reason for any failures. [9644](https://github.com/longhorn/longhorn/issues/9644) - @yangchiu @james-munson - [BACKPORT][v1.6.4][BUG] All Backups are lost in the Backup Target if the NFS Service Disconnects and Reconnects again [9543](https://github.com/longhorn/longhorn/issues/9543) - @yangchiu @mantissahz - [BACKPORT][v1.6.4][BUG] kubectl drain node is blocked by unexpected orphan engine processes [9443](https://github.com/longhorn/longhorn/issues/9443) - @ejweber ### Misc - [TASK] Fix CVE issues for v1.6.4 [9898](https://github.com/longhorn/longhorn/issues/9898) - @c3y1huang - [TASK] Update base image version to 15.6 for v1.6.4 [10073](https://github.com/longhorn/longhorn/issues/10073) - @c3y1huang - [BACKPORT][v1.6.4][TASK] Install the latest grpc_health_probe at build time [9716](https://github.com/longhorn/longhorn/issues/9716) - @yangchiu @c3y1huang ## Contributors - @COLDTURNIP - @ChanYiLin - @PhanLe1010 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @houhoucoop - @innobead - @james-munson - @mantissahz - @roger-ryao - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.7.0.md ================================================ ## Longhorn v1.7.0 Release Notes This latest version of Longhorn introduces several features, enhancements, and bug fixes that are intended to improve system quality and the overall user experience. Highlights include new V2 Data Engine features, platform-agnostic deployment, high availability, and improvements to data protection, stability, performance, and resilience. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). > [!WARNING] > The Longhorn team has identified [a critical issue](https://github.com/longhorn/longhorn/issues/9267) that affects volume attachment in Longhorn v1.7.0. We are currently working on releasing v1.7.1, which will be available soon. If your Longhorn cluster contains `engine` resources with names in the format `-e-<8-char random id>` that are created before v1.5.2 and v1.4.4, please put v1.7.0 upgrade on hold until v1.7.1 is available. > > You can use the following command to check if it is safe to upgrade your Longhorn cluster to v1.7.0: > > ``` > [ $(kubectl -n longhorn-system get engines.longhorn.io -o name | grep -E '\-e\-[a-z0-9]{8}$' | wc -l) -gt 0 ] && echo "Please hold off on upgrading to v1.7.0 until v1.7.1 is available." || echo "Safe to upgrade to v1.7.0." > ``` ## Deprecation & Incompatibilities The functionality of the [environment check script](https://github.com/longhorn/longhorn/blob/v1.7.x/scripts/environment_check.sh) overlaps with that of the Longhorn CLI, which is available starting with v1.7.0. Because of this, the script is deprecated in v1.7.0 and is scheduled for removal in v1.8.0. For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.7.0/deploy/important-notes/) in the Longhorn documentation. ## Primary Highlights ### New V2 Data Engine Features Although the V2 Data Engine is still considered a preview feature in this release, the core functions have been significantly enhanced. - [Online replica rebuilding](https://github.com/longhorn/longhorn/issues/7199): Allows Longhorn to rebuild replicas while the volume is running without any IO interruptions. - [Filesystem trim](https://github.com/longhorn/longhorn/issues/7534): Reclaims unused space more effectively. - [Block-type disk support for SPDK AIO, NVMe and VirtIO Bdev drivers](https://github.com/longhorn/longhorn/issues/7672): Broadens the range of compatible hardware and improves performance in high-demand environments. - [V2 volume support for data plane live upgrade](https://github.com/longhorn/longhorn/issues/6001): Enables upgrading of the spdk_tgt process, which handles IO operations, without any downtime. Coverage of the live upgrade function will be extended to the control plane in a future release. The Longhorn team will continue to develop features for the V1 Data Engine and to prepare the V2 Data Engine for use in all types of environments. ### Platform-Agnostic Deployment Longhorn is designed to seamlessly operate on general-purpose Linux distributions and certain container-optimized operating systems. v1.7.0 provides robust and efficient persistent storage solutions for Kubernetes clusters running on [Container-Optimized OS (COS)](https://github.com/longhorn/longhorn/issues/6165). ### High Availability v1.7.0 introduces features that enhance system resilience and address potential single points of failure. - [High availability for backing images](github.com/longhorn/longhorn/issues/2856): Mitigates risks associated with image failures. - [RWX volumes fast failover](https://github.com/longhorn/longhorn/issues/6205): Enables rapid detection of and response to Share Manager pod failures (independent of the timing and sequence of Kubernetes node failures). ### Data Protection Starting with v1.7.0, Longhorn supports [periodic and on-demand full backups](https://github.com/longhorn/longhorn/issues/7070) that help reduce the likelihood of backup corruption and enhance the overall reliability of the backup process. ### Scheduling The [replica auto-balancing](https://github.com/longhorn/longhorn/issues/4105) feature was enhanced to address disk space pressure from growing volumes. These enhancements reduce manual intervention by automatically rebalancing replicas under disk pressure and improve performance with faster replica rebuilding through local file copying. ### Storage Network The storage network now [supports RWX volumes]((https://github.com/longhorn/longhorn/issues/8184)) with network segregation, enabling dedicated traffic lanes for storage operations. ### Longhorn CLI The [Longhorn CLI](https://github.com/longhorn/longhorn/issues/7927), which is the official Longhorn command line tool, is introduced in v1.7.0. This tool interacts with Longhorn by creating Kubernetes custom resources (CRs) and executing commands inside a dedicated pod for in-cluster and host operations. Usage scenarios include installation, operations such as exporting replicas, and troubleshooting. ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.7.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.7.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.6.x to v1.7.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.7.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Highlights - [FEATURE] Longhorn block-type disks supports different SPDK disk bdev drivers [7672](https://github.com/longhorn/longhorn/issues/7672) - @derekbit @roger-ryao - [FEATURE] Volume cloning enhancement [2907](https://github.com/longhorn/longhorn/issues/2907) - @yangchiu @PhanLe1010 - [FEATURE] Share manager HA - Experimental [6205](https://github.com/longhorn/longhorn/issues/6205) - @PhanLe1010 @james-munson @roger-ryao - [FEATURE] v2 volume replica online rebuilding [7199](https://github.com/longhorn/longhorn/issues/7199) - @shuo-wu @chriscchien - [FEATURE] v2 volume supports live upgrade for data plane [6001](https://github.com/longhorn/longhorn/issues/6001) - @derekbit @chriscchien - [IMPROVEMENT] Use `fsfreeze` instead of `sync` before snapshot [2187](https://github.com/longhorn/longhorn/issues/2187) - @ejweber @chriscchien - [FEATURE] Auto-balancing volumes between nodes & disks [4105](https://github.com/longhorn/longhorn/issues/4105) - @yangchiu @c3y1huang - [FEATURE] HA backing image [2856](https://github.com/longhorn/longhorn/issues/2856) - @ChanYiLin @chriscchien - [FEATURE] Create Longhorn CLI focusing on non CR resource operations [7927](https://github.com/longhorn/longhorn/issues/7927) - @c3y1huang @chriscchien - [FEATURE] Support periodic or on-demand full backups to enhance backup reliability [7070](https://github.com/longhorn/longhorn/issues/7070) - @ChanYiLin @chriscchien - [IMPROVEMENT] Improve rebuilding is canceled if it takes longer than 24 hours [2765](https://github.com/longhorn/longhorn/issues/2765) - @PhanLe1010 @chriscchien - [UI][FEATURE] Support volume cloning using UI [8741](https://github.com/longhorn/longhorn/issues/8741) - @a110605 @chriscchien - [UI][FEATURE] Add backing image encryption and clone support [8789](https://github.com/longhorn/longhorn/issues/8789) - @a110605 @mantissahz - [FEATURE] Support volume encryption for (encrypted) backing image volumes [7051](https://github.com/longhorn/longhorn/issues/7051) - @ChanYiLin @roger-ryao - [FEATURE] Container-Optimized OS support [6165](https://github.com/longhorn/longhorn/issues/6165) - @yangchiu @c3y1huang ### Features - [FEATURE] v2 volume supports filesystem trim [7534](https://github.com/longhorn/longhorn/issues/7534) - @derekbit @chriscchien - [FEATURE] Add nodeSelector as a parameter to VolumeSnapshotClass for backing images [6526](https://github.com/longhorn/longhorn/issues/6526) - @ChanYiLin @chriscchien - [FEATURE] Add additional monitoring settings to ServiceMonitor resource. [8142](https://github.com/longhorn/longhorn/issues/8142) - @yangchiu @ejweber @kejrak - [FEATURE] Support storage network for RWX volumes [8184](https://github.com/longhorn/longhorn/issues/8184) - @c3y1huang @chriscchien - [FEATURE] Add BackupBackingImage UI [7541](https://github.com/longhorn/longhorn/issues/7541) - @a110605 @mantissahz - [FEATURE][UI] Add parameters support to the Backup and RecurringJob [8291](https://github.com/longhorn/longhorn/issues/8291) - @a110605 @chriscchien - [UI][FEATURE] Add Fields to Backing Image Creation and Update [8485](https://github.com/longhorn/longhorn/issues/8485) - @a110605 @chriscchien - [FEATURE] Change parent of a logical volume for volume rebuild [5866](https://github.com/longhorn/longhorn/issues/5866) - @DamiaSan ### Improvements - [IMPROVEMENT] Restore Latest Backup should be applied with BackingImage name value [7560](https://github.com/longhorn/longhorn/issues/7560) - @a110605 @roger-ryao - [IMPROVEMENT] Remove Offline Rebuilding from UI on master and v1.7.0 [9090](https://github.com/longhorn/longhorn/issues/9090) - @a110605 @derekbit @chriscchien - [IMPROVEMENT] Add a label and selector for webhooks to longhorn-manager pod manifest. [8803](https://github.com/longhorn/longhorn/issues/8803) - @james-munson @roger-ryao - [IMPROVEMENT] Fix outdated environment check in longhorn manager [7571](https://github.com/longhorn/longhorn/issues/7571) - @mantissahz @roger-ryao - [IMPROVEMENT] v2 volume snapshot supports `UserCreated` flag [7578](https://github.com/longhorn/longhorn/issues/7578) - @DamiaSan @roger-ryao - [IMPROVEMENT] Undocumented traffic on port 8503 [9076](https://github.com/longhorn/longhorn/issues/9076) - @derekbit - [IMPROVEMENT] Cannot expand a volume created by Longhorn UI [6446](https://github.com/longhorn/longhorn/issues/6446) - @mantissahz @chriscchien - [IMPROVEMENT] Minor log improvement [6396](https://github.com/longhorn/longhorn/issues/6396) - @Vicente-Cheng - [IMPROVEMENT] Refactor the usage of timer reset in SPDK JSON-RPC read method [7297](https://github.com/longhorn/longhorn/issues/7297) - @shuo-wu - [IMPROVEMENT] Update sizes in Engine and Volume resources less frequently [8076](https://github.com/longhorn/longhorn/issues/8076) - @ejweber @chriscchien - [IMPROVEMENT] Docker build of instance-manager should error out if the execution of SPDK pkgdep.sh fails [8063](https://github.com/longhorn/longhorn/issues/8063) - @derekbit @roger-ryao - [IMPROVEMENT] Enable spdk_tgt debug log for helping debug [7939](https://github.com/longhorn/longhorn/issues/7939) - @derekbit @chriscchien - [BUG] Do not terminate nfs-ganesha in share-manager pod after failing to access recovery backend [8345](https://github.com/longhorn/longhorn/issues/8345) - @derekbit @chriscchien - [IMPROVEMENT] Fall back to a running instance-manager if a default is not available [8464](https://github.com/longhorn/longhorn/issues/8464) - @derekbit @chriscchien - [IMPROVEMENT] Only sync log settings to running instance manager pod [8466](https://github.com/longhorn/longhorn/issues/8466) - @derekbit - [IMPROVEMENT] Expose local logical volume and attach it to local node [8551](https://github.com/longhorn/longhorn/issues/8551) - @derekbit @chriscchien - [IMPROVEMENT] Update nfs-ganesha to v5.9 [9008](https://github.com/longhorn/longhorn/issues/9008) - @derekbit @chriscchien - [IMPROVEMENT] Pre-pull images (share-manager image and instance-manager image) on each Longhorn node [8376](https://github.com/longhorn/longhorn/issues/8376) - @mantissahz @chriscchien - [IMPROVEMENT] Add websocket for backup backing image [8849](https://github.com/longhorn/longhorn/issues/8849) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Allow users to request backup volume update [7982](https://github.com/longhorn/longhorn/issues/7982) - @mantissahz @chriscchien - [IMPROVEMENT] Problems mounting XFS volume clones / restored snapshots [8796](https://github.com/longhorn/longhorn/issues/8796) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Do not start any instance-manager pods for the v2 data engine on a node if one is already running [8456](https://github.com/longhorn/longhorn/issues/8456) - @derekbit @roger-ryao - [IMPROVEMENT] Auto cleanup related Snapshot when removing Backup [8365](https://github.com/longhorn/longhorn/issues/8365) - @FrankYang0529 @roger-ryao - [IMPROVEMENT] Improve environment_check script for NFS protocol bug and the host system self diagnosis [7931](https://github.com/longhorn/longhorn/issues/7931) - @james-munson @roger-ryao - [UI][IMPROVEMENT] Tweak some minor UI issues [8646](https://github.com/longhorn/longhorn/issues/8646) - @a110605 @roger-ryao - [IMPROVEMENT] environment_check.sh should check for the iscsi_tcp kernel module [8697](https://github.com/longhorn/longhorn/issues/8697) - @tserong @roger-ryao - [IMPROVEMENT] Improve and simplify chart values.yaml [5089](https://github.com/longhorn/longhorn/issues/5089) - @yangchiu @ChanYiLin - [IMPROVEMENT] The client-go rest client rate limit inside the csi sidecar component might be too small (csi-provisioner, csi-attacjer. csi-snappshotter, csi-attacher) [8699](https://github.com/longhorn/longhorn/issues/8699) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Always update the built-in installed system packages when building component images [8721](https://github.com/longhorn/longhorn/issues/8721) - @c3y1huang @roger-ryao - [UI][IMPROVEMENT] Improve the UX of updating danger zone settings [8071](https://github.com/longhorn/longhorn/issues/8071) - @a110605 @roger-ryao - [IMPROVEMENT] Prevent unnecessary updates of instanceManager status [8420](https://github.com/longhorn/longhorn/issues/8420) - @derekbit @chriscchien - [IMPROVEMENT] Longhor Manager Flood with "Failed to get engine proxy of ... cannot get client for engine" Message [8266](https://github.com/longhorn/longhorn/issues/8266) - @derekbit @chriscchien - [IMPROVEMENT] Add setting value limits in `SettingDefinition` [7441](https://github.com/longhorn/longhorn/issues/7441) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Investigate performance bottleneck in v1 data path [8436](https://github.com/longhorn/longhorn/issues/8436) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Disable revision counter by default [8563](https://github.com/longhorn/longhorn/issues/8563) - @PhanLe1010 @chriscchien - [IMPROVEMENT] BackingImage UI improvement [7293](https://github.com/longhorn/longhorn/issues/7293) - @a110605 @roger-ryao - [IMPROVEMENT] Avoid misleading log messages in longhorn manager while syncing danger zone settings [7797](https://github.com/longhorn/longhorn/issues/7797) - @mantissahz @chriscchien - [IMPROVEMENT] Add setting to configure support bundle timeout for node bundle collection [8623](https://github.com/longhorn/longhorn/issues/8623) - @c3y1huang @roger-ryao - [IMPROVEMENT] RWX volume scheduling [7872](https://github.com/longhorn/longhorn/issues/7872) - @derekbit @chriscchien - [IMPROVEMENT] Expose virtual size of qcow2 backing images [7923](https://github.com/longhorn/longhorn/issues/7923) - @tserong @roger-ryao - [IMPROVEMENT] Saving Settings page changes [7497](https://github.com/longhorn/longhorn/issues/7497) - @a110605 @roger-ryao - [IMPROVEMENT] UI does not find credentials for Backblaze B2 s3-compatible storage for backup [8462](https://github.com/longhorn/longhorn/issues/8462) - @mantissahz - [IMPROVEMENT] Add documents for Longhorn deployed with GitOps [6948](https://github.com/longhorn/longhorn/issues/6948) - @yangchiu - [UI][IMPROVEMENT] Allow users to request backup volume update [8501](https://github.com/longhorn/longhorn/issues/8501) - @a110605 - [IMPROVEMENT] Record instance-manager name for a block disksin `node.spec.diskStatus` [8458](https://github.com/longhorn/longhorn/issues/8458) - @derekbit @chriscchien - [TASK] Mirror the `quay.io/openshift/origin-oauth-proxy` image to Longhorn repo similar to what we are doing for CSI sidecar images [8329](https://github.com/longhorn/longhorn/issues/8329) - @PhanLe1010 @roger-ryao - [IMPROVEMENT] Helm Chart: Support Gateway API and improve Ingress [7889](https://github.com/longhorn/longhorn/issues/7889) - @e3b0c442 @mantissahz @roger-ryao - [IMPROVEMENT] Upgrade support bundle kit version to v0.0.36 [8159](https://github.com/longhorn/longhorn/issues/8159) - @c3y1huang @roger-ryao - [IMPROVEMENT] Cannot read/write to block volume when the container is run as non-root [8088](https://github.com/longhorn/longhorn/issues/8088) - @PhanLe1010 @chriscchien - [IMPROVEMENT] Change support-bundle-manager image pull policy to PullIfNotPresent [7998](https://github.com/longhorn/longhorn/issues/7998) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Use HEAD instead of a GET to fetch the `Content-Length` of an resource via URL [7892](https://github.com/longhorn/longhorn/issues/7892) - @votdev @chriscchien - [IMPROVEMENT] Remove startup probe of CSI driver after liveness probe conn fix ready [7428](https://github.com/longhorn/longhorn/issues/7428) - @ejweber @chriscchien - [IMPROVEMENT] Implement TLS to proxy client [3975](https://github.com/longhorn/longhorn/issues/3975) - @c3y1huang - [DOC] Snapshot Maximum Count setting is missing from the Settings Reference page [7894](https://github.com/longhorn/longhorn/issues/7894) - @FrankYang0529 ### Bug Fixes - [BUG] v2 Volume Filesystem Trim doesn't work without enabling `uio_pci_generic` [9182](https://github.com/longhorn/longhorn/issues/9182) - @derekbit - [BUG] Backup CR Stuck in Deletion if the Source PVC is already removed [9252](https://github.com/longhorn/longhorn/issues/9252) - @mantissahz @chriscchien - [BUG] v2 volume workload pod failed mount with error: UNEXPECTED INCONSISTENCY [9196](https://github.com/longhorn/longhorn/issues/9196) - @derekbit @shuo-wu @chriscchien - [BUG] `longhornctl` doesn't work on `amazon linux` and `sle-micro` [9139](https://github.com/longhorn/longhorn/issues/9139) - @mantissahz @chriscchien - [BUG] v2 volume could get stuck in attaching/detaching loop if deleting replica while replica rebuilding [9190](https://github.com/longhorn/longhorn/issues/9190) - @yangchiu @derekbit @shuo-wu @chriscchien - [BUG] Networking between longhorn-csi-plugin and longhorn-manager is broken after upgrading Longhorn to 1.7.0-rc3 [9223](https://github.com/longhorn/longhorn/issues/9223) - @PhanLe1010 @roger-ryao - [BUG] Longhorn didn't choose to rebuild on an existing replica if there is a scheduling failed replica after [1992](https://github.com/longhorn/longhorn/issues/1992) - @yangchiu @c3y1huang - [BUG] test case `test_system_backup_and_restore_volume_with_backingimage` failed on sle-micro ARM64 [9209](https://github.com/longhorn/longhorn/issues/9209) - @ChanYiLin - [BUG] v2 volume size could become 0 if deleting replica while replica rebuilding [9191](https://github.com/longhorn/longhorn/issues/9191) - @derekbit @shuo-wu @chriscchien - [BUG] Degraded v2 volume write error after replica rebuild [9166](https://github.com/longhorn/longhorn/issues/9166) - @shuo-wu @chriscchien - [BUG] SPDK NVMe bdev is unable to create on some NVMe disks (on Equinix platform) [8813](https://github.com/longhorn/longhorn/issues/8813) - @derekbit @roger-ryao - [BUG] Node block-type disk is unable to unbind from userspace driver after failed to add it to node disk [9126](https://github.com/longhorn/longhorn/issues/9126) - @derekbit @roger-ryao - [BUG] Test case test_rwx_delete_share_manager_pod fails after changes for RWX HA. [9081](https://github.com/longhorn/longhorn/issues/9081) - @yangchiu @PhanLe1010 @james-munson - [BUG] RWX volume is stuck in auto-salvage loop forever if volume becomes faulted and RWX fast failover setting is enabled [9089](https://github.com/longhorn/longhorn/issues/9089) - @PhanLe1010 @chriscchien - [BUG]filesystem trim RecurringJob times out (volumes where files are frequently created and deleted) [6868](https://github.com/longhorn/longhorn/issues/6868) - @c3y1huang @roger-ryao - [BUG] Can not revert V2 volume snapshot after upgrade from v1.6.2 to v1.7.0-dev [9054](https://github.com/longhorn/longhorn/issues/9054) - @chriscchien @DamiaSan - [BUG] Share manager controller reconciles tens of thousands of times [9086](https://github.com/longhorn/longhorn/issues/9086) - @ejweber @roger-ryao - [BUG] Non-existing block device results in longhorn-manager to be in Crashloopbackoff state [9073](https://github.com/longhorn/longhorn/issues/9073) - @yangchiu @derekbit - [BUG] v2 snapshot creation time will change in UI from time to time [7641](https://github.com/longhorn/longhorn/issues/7641) - @chriscchien @DamiaSan - [BUG][v1.7.x-head] Test case `test_support_bundle_should_not_timeout` support bundle cleanup failed [9055](https://github.com/longhorn/longhorn/issues/9055) - @yangchiu @mantissahz - [BUG][v1.7.0-rc1] Clone DR volume result in fail [9016](https://github.com/longhorn/longhorn/issues/9016) - @a110605 @PhanLe1010 @chriscchien - [BUG] longhorn rwx volume fails to mount on first pod [6857](https://github.com/longhorn/longhorn/issues/6857) - @james-munson - [BUG][v1.7.0-rc1] Test case `test_csi_encrypted_block_volume` failed even though `cryptsetup` has been installed [9000](https://github.com/longhorn/longhorn/issues/9000) - @ejweber @roger-ryao - [BUG] System restore with backing image could fail due to backing image checksum mismatch [9041](https://github.com/longhorn/longhorn/issues/9041) - @ChanYiLin @roger-ryao - [BUG] Longhorn v.1.7.0-rc1 on ARM, the v2-data-engine failed to enable the instance-manager pod [9004](https://github.com/longhorn/longhorn/issues/9004) - @derekbit @roger-ryao - [BUG][v1.7.x-head] Test case `test_engine_image_not_fully_deployed_perform_auto_upgrade_engine` failed due to engine image unable to deploy on one of nodes [9038](https://github.com/longhorn/longhorn/issues/9038) - @yangchiu @mantissahz - [BUG][v1.7.x-head] Test case `test_allow_volume_creation_with_degraded_availability_csi` failed due to inconsistent `Scheduled` status [9035](https://github.com/longhorn/longhorn/issues/9035) - @yangchiu @ejweber - [BUG] V2 volume snapshot creation time disappear after upgrade from v1.6.2 to v1.7.0-dev [9045](https://github.com/longhorn/longhorn/issues/9045) - @chriscchien @DamiaSan - [BUG] RWX pod stuck in hanging forever after a node shutdown/reboot [9022](https://github.com/longhorn/longhorn/issues/9022) - @yangchiu @c3y1huang - [BUG][v1.7.0-rc1] Workload unable to recover after kubelet restart with error: MountVolume.MountDevice failed [9014](https://github.com/longhorn/longhorn/issues/9014) - @yangchiu @c3y1huang - [BUG][v1.7.0-rc1] Require unexpected long time to remount RWX volume after share manager restarted [8999](https://github.com/longhorn/longhorn/issues/8999) - @yangchiu @c3y1huang - [BUG][v1.7.x] V2 volume cannot detach after upgrade if a recurring job was set before the upgrade [9032](https://github.com/longhorn/longhorn/issues/9032) - @derekbit @chriscchien - [BUG] Pod auto-deletion may cause thousands of logs [9019](https://github.com/longhorn/longhorn/issues/9019) - @yangchiu @ejweber - [BUG] HA Volume Migration: Volume does not auto-attach to another node after turning off the original node [9039](https://github.com/longhorn/longhorn/issues/9039) - @ejweber @roger-ryao - [BUG][UI][v1.7.0-rc1] Instance manager image search by state not work properly [9010](https://github.com/longhorn/longhorn/issues/9010) - @a110605 @chriscchien - [BUG] The check of longhorn-cli for spdk environment is set to 1024 MiB [8994](https://github.com/longhorn/longhorn/issues/8994) - @derekbit @mantissahz - [BUG] Encrypted volume can't be mounted to the workload [9002](https://github.com/longhorn/longhorn/issues/9002) - - [BUG] The volume remain in attached state even if detach action is called on the volume which is in migration process and old engine node is powered down. [3401](https://github.com/longhorn/longhorn/issues/3401) - @ejweber @chriscchien - [BUG][UI] Snapshots and Backups content not alignment in volume detail page of a restored v2 volume [8964](https://github.com/longhorn/longhorn/issues/8964) - @derekbit @roger-ryao - [BUG] Can not change `storage-network-for-rwx-volume-enabled` when RWX volume attached [8979](https://github.com/longhorn/longhorn/issues/8979) - @c3y1huang @chriscchien - [BUG] Undocumented changes in migration behavior since v1.4.x [8735](https://github.com/longhorn/longhorn/issues/8735) - @yangchiu @ejweber - [BUG] v2 volume data are not sync before taking a snapshot [8977](https://github.com/longhorn/longhorn/issues/8977) - @derekbit @chriscchien - [BUG] Extra recurring jobs being created after volume restoration [8874](https://github.com/longhorn/longhorn/issues/8874) - @ChanYiLin @mantissahz - [BUG] Backing image related test cases failed [8887](https://github.com/longhorn/longhorn/issues/8887) - @ChanYiLin @mantissahz - [BUG] Cannot do a Helm upgrade after PR 2763 [8974](https://github.com/longhorn/longhorn/issues/8974) - @mantissahz @roger-ryao - [BUG] OpenShift 4.15.3 - Lonhorn 1.6.1 - longhorn-ui nginx (13: Permission denied) [8300](https://github.com/longhorn/longhorn/issues/8300) - @mantissahz @roger-ryao - [BUG] Can not create V2 volume(faulted) after upgrade from v1.6.2 to v1.7.0-dev [8967](https://github.com/longhorn/longhorn/issues/8967) - @derekbit @chriscchien - [BUG] Cloned PVC from detached volume will stuck at not ready for workload [3692](https://github.com/longhorn/longhorn/issues/3692) - @PhanLe1010 @chriscchien - [BUG] Volume cloning retry logic causes unnecessary updates to engine CR [8952](https://github.com/longhorn/longhorn/issues/8952) - @PhanLe1010 @roger-ryao - [BUG] controller-gen panic while generating crds.yaml [8901](https://github.com/longhorn/longhorn/issues/8901) - @derekbit @chriscchien - [BUG] Create backup from UI volume page failed [8947](https://github.com/longhorn/longhorn/issues/8947) - @yangchiu @a110605 @chriscchien - [BUG] iscsid - connect to x.x.x.x failed - no route to host [7386](https://github.com/longhorn/longhorn/issues/7386) - @PhanLe1010 @roger-ryao - [BUG] Longhorn upgrade from 1.4.4 to 1.5.5 failing [8578](https://github.com/longhorn/longhorn/issues/8578) - @PhanLe1010 @roger-ryao - [BUG] v2 volume snapshot invalid date [8862](https://github.com/longhorn/longhorn/issues/8862) - @yangchiu @DamiaSan - [BUG] Pod mount took a long time even PV/PVC bound [2590](https://github.com/longhorn/longhorn/issues/2590) - @yangchiu @mantissahz - [BUG] Uninstallation fails if backup exists in error state [3082](https://github.com/longhorn/longhorn/issues/3082) - @mantissahz @chriscchien - [BUG] Executing fstrim while rebuilding causes IO errors [7103](https://github.com/longhorn/longhorn/issues/7103) - @yangchiu @ejweber - [BUG] Workload cannot recover after tainting node [2517](https://github.com/longhorn/longhorn/issues/2517) - @ejweber @chriscchien @roger-ryao - [BUG] Backup marked as "completed" cannot be restored, gzip: invalid header [7687](https://github.com/longhorn/longhorn/issues/7687) - @derekbit @roger-ryao - [BUG] Use config map to update `default-replica-count` won't apply to `default-replica-count.definition.default` if the value equal to current `default-replica-count.value` [7755](https://github.com/longhorn/longhorn/issues/7755) - @james-munson @chriscchien - [BUG] Deadlock between volume migration and upgrade after Longhorn upgrade [7833](https://github.com/longhorn/longhorn/issues/7833) - @ejweber @roger-ryao - [BUG] After perform volume online expand to a unavailable value, expand volume again with a proper value will succeed but actually the storage size not changed. [7841](https://github.com/longhorn/longhorn/issues/7841) - @mantissahz @chriscchien - [BUG] The feature of auto remount read only volume not work on a single node cluster. [7843](https://github.com/longhorn/longhorn/issues/7843) - @ChanYiLin @chriscchien - [BUG] Add Snapshot Maximum Count to the Settings [7906](https://github.com/longhorn/longhorn/issues/7906) - @FrankYang0529 @yardenshoham @roger-ryao - [BUG] BackingImage does not download URL correctly in some situation [7914](https://github.com/longhorn/longhorn/issues/7914) - @votdev @yangchiu - [BUG] The activated DR volume do not contain the latest data. [7945](https://github.com/longhorn/longhorn/issues/7945) - @shuo-wu @roger-ryao - [BUG] A replica may be incorrectly scheduled to a node with an existing failed replica [8043](https://github.com/longhorn/longhorn/issues/8043) - @ejweber @chriscchien - [BUG] Exporting data from the existing replicas has issues in 1.6.0 [8094](https://github.com/longhorn/longhorn/issues/8094) - @ChanYiLin @chriscchien - [BUG] Failed to restore a backup to file by the scripts/restore-backup-to-file.sh with a CIFS backup target. [8126](https://github.com/longhorn/longhorn/issues/8126) - @mantissahz @chriscchien - [BUG] potential risk to unmap a negative number [8235](https://github.com/longhorn/longhorn/issues/8235) - @Vicente-Cheng @roger-ryao - [BUG] Failed to delete a v2 orphan replica [8642](https://github.com/longhorn/longhorn/issues/8642) - @shuo-wu - [BUG] When revision counter is disabled, the engine might choose a replica with a smaller head size to be the source of truth for auto-salvage [8659](https://github.com/longhorn/longhorn/issues/8659) - @PhanLe1010 @chriscchien - [BUG] Uninstallation will fail if invalid backuptarget is set. [8784](https://github.com/longhorn/longhorn/issues/8784) - @mantissahz @chriscchien - [BUG] Unable to create backup recurring job with label [8868](https://github.com/longhorn/longhorn/issues/8868) - @yangchiu @ChanYiLin - [BUG] Test case `test_reuse_failed_replica` failed due to issues with replica-replenishment-wait-interval behavior [8891](https://github.com/longhorn/longhorn/issues/8891) - @yangchiu @c3y1huang - [BUG] Test case `test_allow_volume_creation_with_degraded_availability_restore` failed. Inconsistent replica creation behavior. [8893](https://github.com/longhorn/longhorn/issues/8893) - @yangchiu @c3y1huang - [BUG] Scheduling related test cases fail with wrong Scheduled status [8867](https://github.com/longhorn/longhorn/issues/8867) - @yangchiu @c3y1huang - [BUG] Volume failed to create a new replica after Replica Replenishment Wait Interval [8870](https://github.com/longhorn/longhorn/issues/8870) - @c3y1huang @chriscchien - [BUG] Failed to create a new replica after an existing replica node becomes unschedulable [8872](https://github.com/longhorn/longhorn/issues/8872) - @c3y1huang @roger-ryao - [BUG] Test case `test_migration_with_unscheduled_replica` failed [8873](https://github.com/longhorn/longhorn/issues/8873) - @yangchiu @c3y1huang - [BUG] Unable to create backing image [8877](https://github.com/longhorn/longhorn/issues/8877) - @yangchiu @ChanYiLin - [BUG] Volume failed to delete redundant replicas on the same node [8869](https://github.com/longhorn/longhorn/issues/8869) - @c3y1huang @chriscchien - [BUG] Failed to backup snapshot: Internal error occurred: invalid character 'i' looking for beginning of value [8861](https://github.com/longhorn/longhorn/issues/8861) - @yangchiu @ChanYiLin - [BUG] No replica is created after increased replica count [8871](https://github.com/longhorn/longhorn/issues/8871) - @c3y1huang @chriscchien - [BUG] Helm upgrade from v1.6.2 to v1.7.0-dev failed due to the longhorn-post-upgrade job exceeding the BackoffLimit. [8866](https://github.com/longhorn/longhorn/issues/8866) - @PhanLe1010 @chriscchien - [BUG] Orphan longhorn-engine-manager and longhorn-replica-manager services [8844](https://github.com/longhorn/longhorn/issues/8844) - @PhanLe1010 @chriscchien - [BUG] v2-data-engine-log-level not applied [8865](https://github.com/longhorn/longhorn/issues/8865) - @derekbit @chriscchien - [BUG] Scale replica snapsots warning [4126](https://github.com/longhorn/longhorn/issues/4126) - @yangchiu @ejweber - [BUG] RWX volume is hang on Photon OS [8253](https://github.com/longhorn/longhorn/issues/8253) - @PhanLe1010 @chriscchien - [BUG] spdk_tgt somehow ran into an internal error. [7703](https://github.com/longhorn/longhorn/issues/7703) - @yangchiu @DamiaSan - [BUG] Can not add block disk - Failed to initialize OpenSSL [8846](https://github.com/longhorn/longhorn/issues/8846) - @derekbit @chriscchien - [BUG] longhorn-manager /usr/local/sbin/ volume and noexec configuration [8780](https://github.com/longhorn/longhorn/issues/8780) - @chriscchien @lenglet-k - [IMPROVEMENT] System restore unable to restore volume with backing image [5085](https://github.com/longhorn/longhorn/issues/5085) - @ChanYiLin @roger-ryao - [BUG] backing image isn't restored after restored a system backup [8515](https://github.com/longhorn/longhorn/issues/8515) - @ChanYiLin @roger-ryao - [BUG] System backup failed because backup creation failed. [8650](https://github.com/longhorn/longhorn/issues/8650) - @ChanYiLin @chriscchien - [BUG] instance-manager pod for v2 volume is killed due to a failed liveness probe. [8807](https://github.com/longhorn/longhorn/issues/8807) - @derekbit @chriscchien - [BUG] Replica Auto Balance options under General Setting and under Volume section should have similar case [7530](https://github.com/longhorn/longhorn/issues/7530) - @yangchiu @a110605 - [BUG] longhorn-manager build failed [8400](https://github.com/longhorn/longhorn/issues/8400) - @yangchiu @mantissahz @hookak - [BUG][v1.6.0-rc3] Randomly observed v2 volume the first replica rebuilding failed [7810](https://github.com/longhorn/longhorn/issues/7810) - @DamiaSan - [BUG] Sometimes v2 volume stuck at attaching because engine error [6176](https://github.com/longhorn/longhorn/issues/6176) - @shuo-wu - [BUG] `toomanysnapshots` UI message displays incorrect snapshot count [8668](https://github.com/longhorn/longhorn/issues/8668) - @ejweber @roger-ryao - [BUG in the internal Testing code] Create backup failed: failed lock lock-*.lck type 1 acquisition [7744](https://github.com/longhorn/longhorn/issues/7744) - @yangchiu @ChanYiLin @chriscchien - [BUG] Missed NodeStageVolume after reboot leads to CreateContainerError [8009](https://github.com/longhorn/longhorn/issues/8009) - @ejweber @chriscchien - [BUG] Backing image disk state unknown after unmount disk [6443](https://github.com/longhorn/longhorn/issues/6443) - @ChanYiLin @chriscchien - [BUG] Can not add block disk on longhorn node [8320](https://github.com/longhorn/longhorn/issues/8320) - @derekbit @chriscchien - [BUG] longhorn-engine integration-test fails after introducing the fix of golangci-lint error [8548](https://github.com/longhorn/longhorn/issues/8548) - @derekbit @chriscchien - [BUG] instance-manager is stuck at starting state [8455](https://github.com/longhorn/longhorn/issues/8455) - @derekbit @chriscchien - [BUG] system restore stuck because of the volume/PV/PVC restoration [8601](https://github.com/longhorn/longhorn/issues/8601) - @ChanYiLin @roger-ryao - [BUG] Fix longhorn-manager `TestCleanupRedundantInstanceManagers` [8658](https://github.com/longhorn/longhorn/issues/8658) - @derekbit @roger-ryao - [BUG] Volume cannot attach because of the leftover non-empty volume.status.PendingNodeID after upgrading Longhorn [7994](https://github.com/longhorn/longhorn/issues/7994) - @james-munson @chriscchien - [BUG] LH manager reboots due to the webhook is not ready [8005](https://github.com/longhorn/longhorn/issues/8005) - @ChanYiLin @chriscchien - [BUG] BackupTarget conditions don't reflect connection errors in v1.6.0 [8210](https://github.com/longhorn/longhorn/issues/8210) - @ejweber @chriscchien - [BUG] Longhorn can no longer create XFS volumes smaller than 300 MiB [8488](https://github.com/longhorn/longhorn/issues/8488) - @ejweber @chriscchien - [BUG] Secret for backup not found [8299](https://github.com/longhorn/longhorn/issues/8299) - @mantissahz @roger-ryao - [BUG] Volume failed to create healthy replica after data locality and replica count changed and got stuck in degraded state forever [8522](https://github.com/longhorn/longhorn/issues/8522) - @ejweber @roger-ryao - [BUG] VolumeSnapshot keeps in a non-ready state even related LH snapshot and backup are ready [8618](https://github.com/longhorn/longhorn/issues/8618) - @PhanLe1010 - [BUG] Valid backup secret produces error message: "there is space or new line in AWS_CERT" [7159](https://github.com/longhorn/longhorn/issues/7159) - @mantissahz @chriscchien - [BUG][1.5.0] instance-manager doesn't seem to apply the tolerations [6313](https://github.com/longhorn/longhorn/issues/6313) - @ejweber - [BUG] The README.md in `longhorn-manager` is outdated [6914](https://github.com/longhorn/longhorn/issues/6914) - @mantissahz - [BUG] Instance manager pod consumes high CPU usage [8496](https://github.com/longhorn/longhorn/issues/8496) - @derekbit @roger-ryao - [BUG] Lost connection to unix:///csi/csi.sock [8427](https://github.com/longhorn/longhorn/issues/8427) - @ejweber @roger-ryao - [BUG] share-manager-pvc appears to be leaking memory [8394](https://github.com/longhorn/longhorn/issues/8394) - @derekbit @roger-ryao - [BUG] Longhorn Helm uninstall times out. [8408](https://github.com/longhorn/longhorn/issues/8408) - @ChanYiLin @roger-ryao - [BUG] Disable tls 1.0 and 1.1 on webhook service [8387](https://github.com/longhorn/longhorn/issues/8387) - @ChanYiLin @roger-ryao - [BUG] Longhorn may keep corrupted salvaged replicas and discard good ones [7425](https://github.com/longhorn/longhorn/issues/7425) - @ejweber @roger-ryao - [BUG] v2 volume gets stuck after force deleting one of its replicas [8354](https://github.com/longhorn/longhorn/issues/8354) - @derekbit @chriscchien - [BUG] Decrement the number allocated clusters of a blob after executing UNMAP [8411](https://github.com/longhorn/longhorn/issues/8411) - @DamiaSan - [BUG] gRPC server reflection doesn't work for ProxyEngineService [5752](https://github.com/longhorn/longhorn/issues/5752) - @FrankYang0529 @roger-ryao - [BUG] Replica rebuild failed [8091](https://github.com/longhorn/longhorn/issues/8091) - @yangchiu @shuo-wu - [BUG] persistence.removeSnapshotsDuringFilesystemTrim Helm variable is unreferenced [7909](https://github.com/longhorn/longhorn/issues/7909) - @ejweber @roger-ryao - [BUG] mount volume with rwx mode, frequent error output "kernel: nfs: Deprecated parameter 'intr'" [6599](https://github.com/longhorn/longhorn/issues/6599) - - [BUG] longhorn manager pod fails to start in container-based K3s [5693](https://github.com/longhorn/longhorn/issues/5693) - @ChanYiLin @chriscchien @andrewd-zededa - [BUG] Longhorn api-server PUT request rate [8114](https://github.com/longhorn/longhorn/issues/8114) - @ejweber @roger-ryao - [BUG] DOCS - Incorrect documentation on pre-upgrade checker configuration [8336](https://github.com/longhorn/longhorn/issues/8336) - @yangchiu - [BUG] Can't use longhorn with Generic ephemeral volumes [8198](https://github.com/longhorn/longhorn/issues/8198) - @ejweber @roger-ryao - [BUG] no Pending workload pods for volume xxx to be mounted [8072](https://github.com/longhorn/longhorn/issues/8072) - @c3y1huang @roger-ryao - [BUG] RWX volumes are broken after share-manager base image update [8166](https://github.com/longhorn/longhorn/issues/8166) - @ejweber - [BUG] automatically updating Rancher image mirror list is not working [8066](https://github.com/longhorn/longhorn/issues/8066) - @PhanLe1010 - [BUG] Panic during collecting metrics [8098](https://github.com/longhorn/longhorn/issues/8098) - @derekbit - [BUG] metric `longhorn_volume_robustness` did not reflect detached volume [8139](https://github.com/longhorn/longhorn/issues/8139) - @c3y1huang @chriscchien - [BUG] Deadlock is possible in v1.6.0 instance manager [7919](https://github.com/longhorn/longhorn/issues/7919) - @ejweber @chriscchien - [BUG] ENGINE v2 : disk /dev/xxxx is already used by AIO bdev disk-x [8129](https://github.com/longhorn/longhorn/issues/8129) - @derekbit @chriscchien - [BUG] Volumes stuck upgrading after 1.5.3 -> 1.6.0 upgrade. [7887](https://github.com/longhorn/longhorn/issues/7887) - @yangchiu @ejweber - [BUG][v1.5.4-rc4] Test case test_backuptarget_available_during_engine_image_not_ready failed to wait for backup target available [8045](https://github.com/longhorn/longhorn/issues/8045) - @yangchiu @c3y1huang - [BUG][v1.5.x] Recurring job fails to create backup when volume detached [7937](https://github.com/longhorn/longhorn/issues/7937) - @yangchiu @mantissahz @PhanLe1010 @c3y1huang - [BUG] Fix codespell issues in backing-image-manager [7893](https://github.com/longhorn/longhorn/issues/7893) - @votdev - [BUG][v1.5.4-rc1] Recurring job failed to create/delete backups after node reboot [7854](https://github.com/longhorn/longhorn/issues/7854) - @ChanYiLin @james-munson - [BUG][v1.6.0-rc1] Negative test case failed: Stress Volume Node CPU/Memory When Volume Is Offline Expanding [7707](https://github.com/longhorn/longhorn/issues/7707) - @PhanLe1010 ### Performance - [DOC] Reference Architecture and Sizing Guidelines [2598](https://github.com/longhorn/longhorn/issues/2598) - @yangchiu @PhanLe1010 ### Benchmark - [TASK] Add 1.6.0 performance report [7829](https://github.com/longhorn/longhorn/issues/7829) - @derekbit ### Miscellaneous - [DOC] Last-ditch uninstall cleanup instructions are out of date. [9116](https://github.com/longhorn/longhorn/issues/9116) - @james-munson - [TASK] Fix CVE issues for v1.7.0 (RC3) [9172](https://github.com/longhorn/longhorn/issues/9172) - @c3y1huang - [DOC] Update `Quick Installation` document for `longhornctl` [9075](https://github.com/longhorn/longhorn/issues/9075) - @mantissahz @chriscchien - [TASK] Fix CVE issues for v1.7.0 (RC1) [8976](https://github.com/longhorn/longhorn/issues/8976) - @c3y1huang - [DOC] Improve the usage and official doc for Longhorn CLI [8992](https://github.com/longhorn/longhorn/issues/8992) - @c3y1huang - [TASK] Move longhorn/[samba,nfs-ganesha] to rancher/[samba,nfs-ganesha] before v1.7.0-rc2 [8991](https://github.com/longhorn/longhorn/issues/8991) - @derekbit - [TASK] Bump base images to SLES 15.6 [8273](https://github.com/longhorn/longhorn/issues/8273) - @mantissahz - [TASK] Update the best practice page to mention some broken kernels [8881](https://github.com/longhorn/longhorn/issues/8881) - @yangchiu @PhanLe1010 - [REFACTOR] Move the construction of each backup store service to service constructor [5840](https://github.com/longhorn/longhorn/issues/5840) - @mantissahz @chriscchien - [DOC] Update v2 engine prerequisites [8819](https://github.com/longhorn/longhorn/issues/8819) - @DamiaSan - [TASK] Update nvme-cli to v2.9.1 for v2 data engine [8836](https://github.com/longhorn/longhorn/issues/8836) - @derekbit @roger-ryao - [TASK] Update longhorn-spdk-engine and longhorn-instance-manager to SPDK longhorn-v24.05 [8820](https://github.com/longhorn/longhorn/issues/8820) - @DamiaSan - [TASK] Manage Shallow copy async in go-spdk-helper [8772](https://github.com/longhorn/longhorn/issues/8772) - @DamiaSan - [REFACTOR] Build a method to differentiate process type instead of port number [7393](https://github.com/longhorn/longhorn/issues/7393) - @yangchiu @ChanYiLin - [TASK] Creation of longhorn-v24.05 branch [8595](https://github.com/longhorn/longhorn/issues/8595) - @DamiaSan - [TASK] Upgrade Go version to 1.22 [8337](https://github.com/longhorn/longhorn/issues/8337) - @FrankYang0529 - [FEATURE] Support backing image metrics collection [7563](https://github.com/longhorn/longhorn/issues/7563) - @ChanYiLin @chriscchien - [REFACTOR] move mount point check function to common lib [7394](https://github.com/longhorn/longhorn/issues/7394) - @ChanYiLin @roger-ryao - [TASK] Fix errors when upgrading K8s lib to 0.30 [8056](https://github.com/longhorn/longhorn/issues/8056) - @mantissahz @chriscchien - [DOC] Incorrect and invalid links [8633](https://github.com/longhorn/longhorn/issues/8633) - @jillian-maroket - [TASK] Remove v2 volume offline rebuilding in Longhorn v1.7 [8442](https://github.com/longhorn/longhorn/issues/8442) - @derekbit @chriscchien - [TASK] Performance and scalability report for Longhorn [2986](https://github.com/longhorn/longhorn/issues/2986) - @PhanLe1010 - [TASK] Sync longohorn/spdk codes with upstream codes [7812](https://github.com/longhorn/longhorn/issues/7812) - @DamiaSan - [QUESTION] Safe node shutdown, disk eviction, replicas and orphaned volumes [6921](https://github.com/longhorn/longhorn/issues/6921) - - [DOC] Hugepage-2Mi recommended [7008](https://github.com/longhorn/longhorn/issues/7008) - - [DOC] uninstall longhorn with argocd, the longhorn is still running. [8395](https://github.com/longhorn/longhorn/issues/8395) - - [DOC] Add uninstall and upgrade doc for GitOps solutions [8441](https://github.com/longhorn/longhorn/issues/8441) - @yangchiu - [TASK] Clarify the reason for k8s.io replace directives in go.mod files [8481](https://github.com/longhorn/longhorn/issues/8481) - @ejweber - [DOC] Add a doc for elaborating when will the requesting remount be triggered [7965](https://github.com/longhorn/longhorn/issues/7965) - @ChanYiLin - [TASK] Collect all proto files in longhorn-types repo [6744](https://github.com/longhorn/longhorn/issues/6744) - @FrankYang0529 - Algolia search for DS [8178](https://github.com/longhorn/longhorn/issues/8178) - @jhkrug - Netlify preview site for Docusaurus, provided by the project infrastructure. [8181](https://github.com/longhorn/longhorn/issues/8181) - @jhkrug - DS requires an `authors.yml` file for the KB. [8182](https://github.com/longhorn/longhorn/issues/8182) - @jhkrug - Improve ordering in sections [8220](https://github.com/longhorn/longhorn/issues/8220) - @jhkrug - Improve broken links management [8215](https://github.com/longhorn/longhorn/issues/8215) - @jhkrug - Styling to match the current website in feel [8173](https://github.com/longhorn/longhorn/issues/8173) - @jhkrug - Remove or change Hugo formatting to Docusaurus [8165](https://github.com/longhorn/longhorn/issues/8165) - @jhkrug - [TASK] Downgrade the package `github.com/prometheus/common` [8274](https://github.com/longhorn/longhorn/issues/8274) - @mantissahz - [TASK] SPDK raid merge with upstream [8074](https://github.com/longhorn/longhorn/issues/8074) - @DamiaSan - [DOC] Update fleet document for ignoring more modified crd [8214](https://github.com/longhorn/longhorn/issues/8214) - @yangchiu - [TASK] Generate BackupBackingImage Backend API [8068](https://github.com/longhorn/longhorn/issues/8068) - @ChanYiLin @chriscchien - [DOC] Typo on Longhorn-specific StorageClass parameters [8170](https://github.com/longhorn/longhorn/issues/8170) - - [UI][TASK] Generate BackupBackingImage Backend API [8069](https://github.com/longhorn/longhorn/issues/8069) - - [TASK] Clean up defunct patches and scripts [5945](https://github.com/longhorn/longhorn/issues/5945) - @ejweber - [TASK] Update version file in component repos automatically when starting a new release [7958](https://github.com/longhorn/longhorn/issues/7958) - ## Contributors - @ChanYiLin - @DamiaSan - @FrankYang0529 - @PhanLe1010 - @Vicente-Cheng - @a110605 - @andrewd-zededa - @c3y1huang - @chriscchien - @derekbit - @e3b0c442 - @ejweber - @hookak - @innobead - @james-munson - @jhkrug - @jillian-maroket - @kejrak - @lenglet-k - @mantissahz - @roger-ryao - @shuo-wu - @tserong - @votdev - @yangchiu - @yardenshoham - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.7.1.md ================================================ ## Longhorn v1.7.1 Release Notes Longhorn 1.7.1 introduces several improvements and bug fixes that are intended to improve system quality, resilience, and stability. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.7.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.7.1/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.6.x or v1.7.x (< v1.7.0) to v1.7.1.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.7.1/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities The functionality of the [environment check script](https://github.com/longhorn/longhorn/blob/v1.7.x/scripts/environment_check.sh) overlaps with that of the Longhorn CLI, which is available starting with v1.7.0. Because of this, the script is deprecated in v1.7.0 and is scheduled for removal in v1.8.0. For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.7.1/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.7.1][IMPROVEMENT] Longhorn CLI should install `cryptsetup` [9316](https://github.com/longhorn/longhorn/issues/9316) - @mantissahz @roger-ryao - [BACKPORT][v1.7.1][IMPROVEMENT] Resilience handling for the last replica timeout [9275](https://github.com/longhorn/longhorn/issues/9275) - @ejweber @chriscchien - [BACKPORT][v1.7.1][IMPROVEMENT] Check kernel module `dm_crypt` on host machines [9310](https://github.com/longhorn/longhorn/issues/9310) - @mantissahz ### Bug - [BACKPORT][v1.7.1][BUG] Fix security issues in v1.7.1 RC images [9363](https://github.com/longhorn/longhorn/issues/9363) - @c3y1huang - [BACKPORT][v1.7.1][BUG] should set backing image minNumberOfCopies to 1 when upgrading Longhorn [9353](https://github.com/longhorn/longhorn/issues/9353) - @ChanYiLin @chriscchien - [BACKPORT][v1.7.1][BUG] System Backup Fails and DR Volume Enters Attach-Detach Loop When Volume Backup Policy is Set to `Always` [9333](https://github.com/longhorn/longhorn/issues/9333) - @c3y1huang @roger-ryao - [BACKPORT][v1.7.1][BUG] [Backupstore] Need to close the reader after downloading files for the Azure backup store driver. [9282](https://github.com/longhorn/longhorn/issues/9282) - @yangchiu @mantissahz - [BACKPORT][v1.7.1][BUG] Unable to access azurite backup store by DNS hostname [9341](https://github.com/longhorn/longhorn/issues/9341) - @mantissahz - [BACKPORT][v1.7.1][BUG] v1 volume replica rebuild fail after upgrade from v1.7.0 to v1.7.1-rc1 [9332](https://github.com/longhorn/longhorn/issues/9332) - @PhanLe1010 @chriscchien - [BACKPORT][v1.7.1][BUG] v2-data-engine setting validator doesn't take disabled nodes into account when checking hugepages [9320](https://github.com/longhorn/longhorn/issues/9320) - @tserong @roger-ryao - [BACKPORT][v1.7.1][BUG] Some volumes stuck in "Attaching" state after upgrade to 1.7.0 [9270](https://github.com/longhorn/longhorn/issues/9270) - @ChanYiLin @roger-ryao - [BACKPORT][v1.7.1][BUG] error logs appeared in uninstallation job [9304](https://github.com/longhorn/longhorn/issues/9304) - @ChanYiLin @chriscchien - [BACKPORT][v1.7.1][BUG] Incorrect NFS endpoint after enable/disable storage network for RWX volume [9273](https://github.com/longhorn/longhorn/issues/9273) - @Vicente-Cheng @roger-ryao - [BACKPORT][v1.7.1][BUG] Volume stuck in degraded [9285](https://github.com/longhorn/longhorn/issues/9285) - @PhanLe1010 @chriscchien - [BACKPORT][v1.7.1][BUG] LH fails silently when node has attached volumes [9210](https://github.com/longhorn/longhorn/issues/9210) - @ejweber @chriscchien ## Contributors - @ChanYiLin - @PhanLe1010 - @Vicente-Cheng - @c3y1huang - @chriscchien - @ejweber - @innobead - @mantissahz - @roger-ryao - @tserong - @yangchiu - @jillian-maroket - @yardenshoham - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.7.2.md ================================================ ## Longhorn v1.7.2 Release Notes Longhorn 1.7.2 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.7.2.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.7.2/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.6.x or v1.7.x (< v1.7.0) to v1.7.2.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.7.2/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities The functionality of the [environment check script](https://github.com/longhorn/longhorn/blob/v1.7.x/scripts/environment_check.sh) overlaps with that of the Longhorn CLI, which is available starting with v1.7.0. Because of this, the script is deprecated in v1.7.0 and is scheduled for removal in v1.8.0. For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.7.2/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.7.2][IMPROVEMENT] Allow specify data engine version for the default storageclass during Helm installation [9632](https://github.com/longhorn/longhorn/issues/9632) - @shuo-wu @chriscchien - [BACKPORT][v1.7.2][IMPROVEMENT] Remove mirrored openshift image from Longhorn [9598](https://github.com/longhorn/longhorn/issues/9598) - @derekbit @chriscchien - [BACKPORT][v1.7.2][IMPROVEMENT] Fix contradicting node status events [9326](https://github.com/longhorn/longhorn/issues/9326) - @yangchiu @PhanLe1010 - [BACKPORT][v1.7.2][IMPROVEMENT] Check kernel module `dm_crypt` on host machines [9317](https://github.com/longhorn/longhorn/issues/9317) - @yangchiu @mantissahz ### Bug - [BACKPORT][v1.7.2][BUG] longhornctl install preflight --operating-system=cos failed on COS_CONTAINERD [9666](https://github.com/longhorn/longhorn/issues/9666) - @c3y1huang @chriscchien - [BACKPORT][v1.7.2][BUG] System Restore Stuck at Pending due to Tolerations not Applied [9655](https://github.com/longhorn/longhorn/issues/9655) - @c3y1huang @chriscchien - [BACKPORT][v1.7.2][BUG][v1.7.x] Disks modal broken layout [9633](https://github.com/longhorn/longhorn/issues/9633) - @a110605 @roger-ryao - [BACKPORT][v1.7.2][BUG] Single Replica Node Down test cases fail [9628](https://github.com/longhorn/longhorn/issues/9628) - @yangchiu @c3y1huang - [BACKPORT][v1.7.2][BUG] Test case `Stopped replicas on deleted nodes should not be counted as healthy replicas when draining nodes` fails [9621](https://github.com/longhorn/longhorn/issues/9621) - @yangchiu @derekbit - [BUG] robot test case `Single Replica Node Down Deletion Policy do-nothing With RWO Volume Replica Locate On Replica Node` fails to wait for the volume getting stuck in attaching state [9498](https://github.com/longhorn/longhorn/issues/9498) - @c3y1huang - [BACKPORT][v1.7.2][BUG] All Backups are lost in the Backup Target if the NFS Service Disconnects and Reconnects again [9542](https://github.com/longhorn/longhorn/issues/9542) - @yangchiu @mantissahz - [BACKPORT][v1.7.2][BUG] PV Annotation Isn't Updated After Creating An Oversize Volume [9515](https://github.com/longhorn/longhorn/issues/9515) - @c3y1huang @chriscchien - [BACKPORT][v1.7.2][BUG] Instance manager missing required selector labels after manager crash [9471](https://github.com/longhorn/longhorn/issues/9471) - @c3y1huang @chriscchien - [BACKPORT][v1.7.2][BUG] Engine Upgrade to 1.7.1 fails on volumes with strict-local data locality [9416](https://github.com/longhorn/longhorn/issues/9416) - @yangchiu @james-munson - [BACKPORT][v1.7.2][BUG] Longhorn did not close and open encrypted volumes correctly when the service k3s-agent restarted for a while [9387](https://github.com/longhorn/longhorn/issues/9387) - @yangchiu @mantissahz - [BACKPORT][v1.7.2][BUG] Longhorn keeps resetting my storageClass [9396](https://github.com/longhorn/longhorn/issues/9396) - @yangchiu @mantissahz - [BACKPORT][v1.7.2][BUG] Fix test case test_rwx_delete_share_manager_pod failure after changes to RWX workload restart. [9505](https://github.com/longhorn/longhorn/issues/9505) - @yangchiu @james-munson - [BACKPORT][v1.7.2][BUG] Remove unnecessary restart of RWX workload. [9290](https://github.com/longhorn/longhorn/issues/9290) - @yangchiu @james-munson - [BACKPORT][v1.7.2][BUG] Faulted RWX volume upon creation [9474](https://github.com/longhorn/longhorn/issues/9474) - @yangchiu @james-munson - [BACKPORT][v1.7.2][BUG] Accidentally encountered a single replica volume backup stuck at progress 17% indefinitely after a node rebooted [9312](https://github.com/longhorn/longhorn/issues/9312) - @yangchiu @ChanYiLin - [BACKPORT][v1.7.2][BUG] Longhorn thinks node is unschedulable [9382](https://github.com/longhorn/longhorn/issues/9382) - @c3y1huang @roger-ryao ## Misc - [BACKPORT][v1.7.2][TASK] Fix CVE issues in support bundle [9659](https://github.com/longhorn/longhorn/issues/9659) - @yangchiu @c3y1huang - [BACKPORT][v1.7.2][TASK] Update CSI components to address CVE issues [9563](https://github.com/longhorn/longhorn/issues/9563) - @derekbit @c3y1huang @chriscchien ## Contributors - @ChanYiLin - @PhanLe1010 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @innobead - @james-munson - @mantissahz - @roger-ryao - @shuo-wu - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.7.3.md ================================================ ## Longhorn v1.7.3 Release Notes Longhorn 1.7.3 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before installing Longhorn v1.7.3.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.7.3/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.21 or later before upgrading from Longhorn v1.6.x or v1.7.x (< v1.7.0) to v1.7.3.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.7.3/deploy/upgrade/) in the Longhorn documentation. ## Deprecation & Incompatibilities The functionality of the [environment check script](https://github.com/longhorn/longhorn/blob/v1.7.x/scripts/environment_check.sh) overlaps with that of the Longhorn CLI, which is available starting with v1.7.0. Because of this, the script is deprecated in v1.7.0 and is scheduled for removal in v1.8.0. For information about important changes, including feature incompatibility, deprecation, and removal, see [Important Notes](https://longhorn.io/docs/1.7.3/important-notes/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Feature - [BACKPORT][v1.7.3][FEATURE] Add periodic HugePages (2Mi) configuration check to ensure v2 data engine compatibility [10029](https://github.com/longhorn/longhorn/issues/10029) - @jangseon-ryu @yangchiu ### Improvement - [BACKPORT][v1.7.3][IMPROVEMENT] Settings change validation should go back to using Volume state to determine "are all volumes detached" [10375](https://github.com/longhorn/longhorn/issues/10375) - @yangchiu @james-munson - [BACKPORT][v1.7.3][IMPROVEMENT] Add support for JSON log format configuration in Longhorn components (UI, driver) [10082](https://github.com/longhorn/longhorn/issues/10082) - @chriscchien - [BACKPORT][v1.7.3][IMPROVEMENT] Logging the reason why the instance manager pod is going to be deleted. [9887](https://github.com/longhorn/longhorn/issues/9887) - @derekbit @chriscchien - [BACKPORT][v1.7.3][IMPROVEMENT] Why is it not possible to change the replica count in v2 longhorn volume? [9806](https://github.com/longhorn/longhorn/issues/9806) - @chriscchien - [BACKPORT][v1.7.3][IMPROVEMENT] Check NFS versions in /etc/nfsmount.conf instead [9831](https://github.com/longhorn/longhorn/issues/9831) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.7.3][IMPROVEMENT] Add dmsetup and dmcrypt utilities check in cli [9935](https://github.com/longhorn/longhorn/issues/9935) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.7.3][UI][IMPROVEMENT] Improve the volume size information on UI [9964](https://github.com/longhorn/longhorn/issues/9964) - @houhoucoop @roger-ryao - [BACKPORT][v1.7.3][IMPROVEMENT] Prevent Volume Resize Stuck [9914](https://github.com/longhorn/longhorn/issues/9914) - @c3y1huang @roger-ryao - [BACKPORT][v1.7.3][IMPROVEMENT][UI] Add backupBackingImage table in backup page with tabs [9970](https://github.com/longhorn/longhorn/issues/9970) - @houhoucoop - [BACKPORT][v1.7.3][IMPROVEMENT] Reject strict-local + RWX volume creation [9930](https://github.com/longhorn/longhorn/issues/9930) - @COLDTURNIP @yangchiu - [BACKPORT][v1.7.3][IMPROVEMENT] Configure the log level of other system and user managed components via longhorn manager setting [9617](https://github.com/longhorn/longhorn/issues/9617) - @yangchiu @james-munson - [BACKPORT][v1.7.3][IMPROVEMENT] Change confusing error message to warning level [9917](https://github.com/longhorn/longhorn/issues/9917) - @yangchiu @derekbit - [BACKPORT][v1.7.3][IMPROVEMENT] Building longhorn-manager takes long time [9693](https://github.com/longhorn/longhorn/issues/9693) - @derekbit @chriscchien - [BACKPORT][v1.7.3][IMPROVEMENT] Talos support for environment check in longhorn manager [9723](https://github.com/longhorn/longhorn/issues/9723) - @yangchiu @c3y1huang ### Bug - [BACKPORT][v1.7.3][BUG] Data lost caused by Longhorn CSI plugin doing a wrong filesystem format action in a rare race condition [10417](https://github.com/longhorn/longhorn/issues/10417) - @yangchiu @PhanLe1010 @chriscchien - [BACKPORT][v1.7.3][BUG] kubectl drain node is blocked by unexpected orphan engine processes [10427](https://github.com/longhorn/longhorn/issues/10427) - @yangchiu @PhanLe1010 - [BACKPORT][v1.7.3][BUG] Test case `test_csi_mount_volume_online_expansion` is failing due to unable to expand PVC [10413](https://github.com/longhorn/longhorn/issues/10413) - @yangchiu @c3y1huang - [BACKPORT][v1.7.3][BUG] Workload pod will not be able to move to new node when backup operation is taking a long time [10173](https://github.com/longhorn/longhorn/issues/10173) - @yangchiu - [BUG][v1.7.x] Excessive memory consumption caused by RWX volumes / ganesha.nfsd [8523](https://github.com/longhorn/longhorn/issues/8523) - @james-munson @chriscchien - [BACKPORT][v1.7.3][BUG] WebUI Volumes Disappear and Reappear [10331](https://github.com/longhorn/longhorn/issues/10331) - @PhanLe1010 @chriscchien @houhoucoop - [BACKPORT][v1.7.3][BUG] "Error get size" from "metrics_collector.(*BackupCollector).Collect" on every metric scrape [10362](https://github.com/longhorn/longhorn/issues/10362) - @derekbit @chriscchien - [BACKPORT][v1.7.3][BUG] Engine stuck in "stopped" state, prevent volume attach [9954](https://github.com/longhorn/longhorn/issues/9954) - @ChanYiLin @roger-ryao - [BACKPORT][v1.7.3][BUG] Backup Execution Timeout setting issue in Helm chart [10326](https://github.com/longhorn/longhorn/issues/10326) - @james-munson @chriscchien - [BACKPORT][v1.7.3][BUG] Instability after power failure [10185](https://github.com/longhorn/longhorn/issues/10185) - @yangchiu @james-munson - [BACKPORT][v1.7.3][BUG] CSI plugin pod keep crashing util the backup volume appears when creation a backup via the CSI snapshotter [10024](https://github.com/longhorn/longhorn/issues/10024) - @mantissahz @chriscchien - [BACKPORT][v1.7.3][BUG] insufficient storage;precheck new replica failed after a temporary shutdown of a node [10223](https://github.com/longhorn/longhorn/issues/10223) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.7.3][BUG] longhorn-manager seems to crash rpm-DB on the host by continuously calling rpm -q ... [10022](https://github.com/longhorn/longhorn/issues/10022) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.7.3][BUG] Backup progress should not add block failed to upload to successful count [9793](https://github.com/longhorn/longhorn/issues/9793) - @derekbit @chriscchien - [BACKPORT][v1.7.3][BUG][v1.8.x] Can not create backup, backup become in error state immediately [10180](https://github.com/longhorn/longhorn/issues/10180) - @PhanLe1010 @chriscchien - [BACKPORT][v1.7.3][BUG] Storage doesn't reschedule in v1.7.2 [10109](https://github.com/longhorn/longhorn/issues/10109) - @PhanLe1010 - [BACKPORT][v1.7.3][BUG] Old backups are not cleaned up after timeout [9731](https://github.com/longhorn/longhorn/issues/9731) - @mantissahz @roger-ryao - [BACKPORT][v1.7.3][BUG] UnknowOS Message in Longhorn Node Condition on RHEL [9833](https://github.com/longhorn/longhorn/issues/9833) - @yangchiu @mantissahz @roger-ryao - [BACKPORT][v1.7.3][BUG] volume FailedMount - Input/output error [10005](https://github.com/longhorn/longhorn/issues/10005) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.7.3][BUG] Unable to delete backing image backup through UI [10068](https://github.com/longhorn/longhorn/issues/10068) - @chriscchien @houhoucoop @roger-ryao - [BACKPORT][v1.7.3][BUG] Error notification appears on the volume backup details page [10071](https://github.com/longhorn/longhorn/issues/10071) - @houhoucoop @roger-ryao - [BACKPORT][v1.7.3][BUG] Missing `fromBackup` Parameter in API Request When Restoring Multiple Files from Backup List [10051](https://github.com/longhorn/longhorn/issues/10051) - @a110605 @roger-ryao - [BACKPORT][v1.7.3][BUG] Webhook servers initialization blocks longhorn-manager from running [10055](https://github.com/longhorn/longhorn/issues/10055) - @c3y1huang @chriscchien - [BACKPORT][v1.7.3][BUG] CLI check preflight glosses over absence of NFS installation. [9893](https://github.com/longhorn/longhorn/issues/9893) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.7.3][BUG] Detached Volume Stuck in Attached State During Node Eviction [9810](https://github.com/longhorn/longhorn/issues/9810) - @yangchiu @c3y1huang - [BACKPORT][v1.7.3][BUG] Test case test_node_eviction_multiple_volume failed to reschedule replicas after volume detached [9866](https://github.com/longhorn/longhorn/issues/9866) - @yangchiu @c3y1huang - [BACKPORT][v1.7.3][BUG] DR volume fails to reattach and faulted after node stop and start during incremental restore [9803](https://github.com/longhorn/longhorn/issues/9803) - @c3y1huang @roger-ryao - [BACKPORT][v1.7.3][BUG] Share manager is permanently stuck in stopping/error if we shutdown the node of share manager pod. This makes RWX PVC cannot attach to any new node [9856](https://github.com/longhorn/longhorn/issues/9856) - - [BACKPORT][v1.7.3][BUG] Fail to resize RWX PVC at filesystem resizing step [9738](https://github.com/longhorn/longhorn/issues/9738) - @james-munson - [BACKPORT][v1.7.3][BUG] Failed to inspect the backup backing image information if NFS backup target URL with options [9703](https://github.com/longhorn/longhorn/issues/9703) - @yangchiu @mantissahz - [BACKPORT][v1.7.3][BUG] Pre-upgrade pod should event the reason for any failures. [9643](https://github.com/longhorn/longhorn/issues/9643) - @yangchiu @james-munson ## Misc - [TASK] Fix CVE issues for v1.7.3 [9897](https://github.com/longhorn/longhorn/issues/9897) - @c3y1huang - [BACKPORT][v1.7.3][TASK] Install the latest grpc_health_probe at build time [9715](https://github.com/longhorn/longhorn/issues/9715) - @yangchiu @c3y1huang ## Contributors - @COLDTURNIP - @ChanYiLin - @PhanLe1010 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @houhoucoop - @innobead - @james-munson - @jangseon-ryu - @mantissahz - @roger-ryao - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.8.0.md ================================================ ## Longhorn v1.8.0 Release Notes This latest version of Longhorn introduces several features, enhancements, and bug fixes that are intended to improve system quality and the overall user experience. Highlights include new V2 Data Engine features, multiple backupstores, automatic RWX volume expansion, installation and upgrades via Helm Controller, and V2 Data Engine Support for Talos Linux. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). > [!WARNING] > An incorrect Longhorn image tag (`v1.8.x-head`) was used in the [deployment manifest](https://github.com/longhorn/longhorn/blob/v1.8.0/deploy/longhorn.yaml) and the [Helm chart](https://github.com/longhorn/longhorn/blob/v1.8.0/chart/values.yaml#L40-L65). The correct tag for Longhorn v1.8.0 images is `v1.8.0`. For more information, see [Issue #10336](https://github.com/longhorn/longhorn/issues/10336). > > If you installed or upgraded Longhorn using the deployment manifest or the Helm chart from the [main Longhorn repository](https://github.com/longhorn/longhorn), perform the following actions to resolve the issue: > > - **New installations**: Replace `v1.8.x-head` with `v1.8.0` in the deployment manifest or the Helm chart before deploying Longhorn. > > - **Upgrades**: Replace `v1.8.x-head` with `v1.8.0` in the deployment manifest or Helm chart. Next, upgrade the Longhorn system and update the engine image for volumes that use `v1.8.x-head`. > > This issue does not affect installations and upgrades performed using the [Longhorn Helm repository](https://charts.longhorn.io). For more details, refer to the [Install with Helm](https://longhorn.io/docs/1.8.0/deploy/install/install-with-helm/) section of the official documentation. > [!IMPORTANT] The CSI external-snapshotter was upgraded to v8.2.0. Ensure that all clusters are running Kubernetes v1.25 or later before upgrading to Longhorn v1.8.0 or a later version. ## Deprecation & Incompatibilities - The default block size for block-type disks in earlier Longhorn releases is 4096 bytes. However, 512 bytes is more commonly used and aligns with the V1 Data Engine's configuration. Additionally, the 4096-byte block size is incompatible with backing images generated by the V1 Data Engine. To address these concerns, the default block size was changed to 512 bytes. If you have existing V2 volumes, perform the following steps: 1. Back up the V2 volumes. 2. Remove the V2 volumes. 3. Delete the block-type disk with a 4096-byte block size from `node.spec.disks`. 4. Erase the old data on the block-type disk using tools such as dd. 5. Add the disk again to `node.spec.disks` with the updated configuration. 6. Restore the V2 volumes. For more information, see [#10053](https://github.com/longhorn/longhorn/issues/10053). - A V2 volume data corruption issue that affects earlier Longhorn releases has been resolved in v1.8.0. The issue involves potential continual changes to the checksum of files in a V2 volume with multiple replicas. This occurs because SPDK allocates clusters without initialization, leading to data inconsistencies across replicas. The varying data read from the volume can result in data corruption and broken backups. For more information, see [#10035](https://github.com/longhorn/longhorn/issues/10135). ## Primary Highlights ### New V2 Data Engine Features Although the V2 Data Engine is still considered an experimental feature in this release, the core functions have been significantly enhanced. - [Configurable CPU cores](https://longhorn.io/docs/1.8.0/v2-data-engine/features/configurable-cpu-cores): Support the global and node-specific configuration options provide greater control and flexibility for optimizing performance and resource allocation. - [Disaster recovery (DR) volumes](https://github.com/longhorn/longhorn/issues/6613): Designed to store data in a backup cluster in the event of a failure in the main cluster. DR volumes enhance the resiliency of Longhorn volumes by ensuring data can be quickly restored in case of cluster outages. - [Auto-salvage volumes](https://github.com/longhorn/longhorn/issues/8430): Automatically repair volumes in the event of a failure. - [Live migration](https://github.com/longhorn/longhorn/issues/6361): Allow for the migration of volumes from one node to another without interrupting the services using those volumes. - [Volume encryption](https://github.com/longhorn/longhorn/issues/7355): Ensure data stored in volumes is protected through encryption in transit and at rest. - [Delta replica rebuilding using snapshot checksum](https://github.com/longhorn/longhorn/issues/9488). - [Backing image update and download](https://github.com/longhorn/longhorn/issues/6341). ### Multiple Backupstores and Default Backup Target Starting with v1.8.0, Longhorn allows you to use multiple backupstores for storing backups of Longhorn volumes. Longhorn v1.8.0 also creates a default backup target (`default`) during installation and upgrades. The default backup target is used for system backups and volumes that were created without an assigned backup target name. [Demo](https://www.youtube.com/watch?v=cvsz6YLDLrM) | [Documentation](https://longhorn.io/docs/1.8.0/snapshots-and-backups/backup-and-restore/set-backup-target/) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/5411) ### Automatic RWX Volume Expansion Longhorn v1.8.0 supports fully automatic online expansion of RWX volumes without the need to scale down the workload or apply manual commands. To use this feature, ensure that the v1.8.0 versions of Longhorn Manager, Share Manager, and the CSI plugin are all running. [Documentation](https://longhorn.io/docs/1.8.0/nodes-and-volumes/volumes/expansion/#rwx-volume) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/9736) ### Helm Controller You can now install and upgrade Longhorn on clusters running RKE2 or K3s using the Helm Controller that is built into those distributions. The Helm Controller manages Helm charts using a HelmChart Custom Resource Definition (CRD), which contains most of the options that would normally be passed to the Helm command-line tool. [Documentation](https://longhorn.io/docs/1.8.0/deploy/install/install-with-helm-controller/) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/9506) ### V2 Data Engine Support for Talos Linux Longhorn v1.8.0 supports usage of V2 volumes in Talos Linux clusters. To use this feature, ensure that all nodes meet the V2 Data Engine prerequisites. [Documentation](https://longhorn.io/docs/1.8.0/advanced-resources/os-distro-specific/talos-linux-support#v2-data-engine) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/7791) ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.8.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.8.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.7.x to v1.8.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.8.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ### Highlight - [FEATURE] Support encrypted v2 volumes [7355](https://github.com/longhorn/longhorn/issues/7355) - @mantissahz @chriscchien - [FEATURE] Support v2 volume delta replica rebuilding based on snapshot checksum: data plane and control plane [9488](https://github.com/longhorn/longhorn/issues/9488) - @shuo-wu @roger-ryao - [FEATURE] Multiple backup stores support [5411](https://github.com/longhorn/longhorn/issues/5411) - @mantissahz @roger-ryao - [FEATURE] Support v2 volume delta replica rebuilding based on snapshot checksum: SPDK part [5573](https://github.com/longhorn/longhorn/issues/5573) - @DamiaSan @roger-ryao - [IMPROVEMENT] v2 volume supports data locality [9371](https://github.com/longhorn/longhorn/issues/9371) - @derekbit @roger-ryao - [FEATURE] SPDK volume supports live migration [6361](https://github.com/longhorn/longhorn/issues/6361) - @PhanLe1010 @chriscchien @roger-ryao - [FEATURE] Support backing image for v2 volume [6341](https://github.com/longhorn/longhorn/issues/6341) - @yangchiu @ChanYiLin @chriscchien - [FEATURE] Make CPU core configurable for v2 data engine [8835](https://github.com/longhorn/longhorn/issues/8835) - @yangchiu @derekbit - [FEATURE] Support/verify installing Longhorn Helm chart using helm-controller which is built in k3s and rke2 [9506](https://github.com/longhorn/longhorn/issues/9506) - @yangchiu @james-munson - [FEATURE] v2 volume supports auto salvage [8430](https://github.com/longhorn/longhorn/issues/8430) - @c3y1huang @chriscchien - [FEATURE] v2 volumes supports DR volume [6613](https://github.com/longhorn/longhorn/issues/6613) - @c3y1huang @roger-ryao - [FEATURE] Talos support for v2 data engine [7791](https://github.com/longhorn/longhorn/issues/7791) - @yangchiu @c3y1huang ### Feature - [FEATURE] Support Data Engine Option For Backing Image in CSI flow [10084](https://github.com/longhorn/longhorn/issues/10084) - @ChanYiLin @roger-ryao - [FEATURE] Missing option to modify backup-task retain value from UI while editing recurring job [8810](https://github.com/longhorn/longhorn/issues/8810) - @houhoucoop @roger-ryao - [UI][FEATURE] Multiple backup stores support [8647](https://github.com/longhorn/longhorn/issues/8647) - @a110605 @roger-ryao - [UI][FEATURE] Support V2 Backing Image in UI [9880](https://github.com/longhorn/longhorn/issues/9880) - @yangchiu @chriscchien @houhoucoop - [FEATURE] Add periodic HugePages (2Mi) configuration check to ensure v2 data engine compatibility [9983](https://github.com/longhorn/longhorn/issues/9983) - @yangchiu @jangseon-ryu - [FEATURE][UI] Fill in the secret and secret namespace when restoring the BackupBackingImage [9490](https://github.com/longhorn/longhorn/issues/9490) - @chriscchien @houhoucoop - [FEATURE] Add kernel module check for data engine v2 [9915](https://github.com/longhorn/longhorn/issues/9915) - @jangseon-ryu @roger-ryao - [FEATURE]: Automatic online filesystem resize for RWX volumes. [9736](https://github.com/longhorn/longhorn/issues/9736) - @yangchiu @james-munson - [FEATURE] Add recurring job label to backup metrics or empty value [9429](https://github.com/longhorn/longhorn/issues/9429) - @ChanYiLin @roger-ryao - [FEATURE] Longhorn CLI supports darwin platform [9532](https://github.com/longhorn/longhorn/issues/9532) - @derekbit @chriscchien ### Improvement - [IMPROVEMENT] Bump nfs-ganesha in longhorn-share-manager from v6.3 to v6.5 [10194](https://github.com/longhorn/longhorn/issues/10194) - @derekbit @chriscchien - [IMPROVEMENT] V2 Data engine avoids changing the data of a lvol when decoupling it from [9922](https://github.com/longhorn/longhorn/issues/9922) - @DamiaSan @roger-ryao - [IMPROVEMENT] Change the AIO disk block size to 512 bytes for v2 data engine [10053](https://github.com/longhorn/longhorn/issues/10053) - @derekbit @chriscchien - [IMPROVEMENT] No need to truncate block-type disk's StorageAvailable value [10121](https://github.com/longhorn/longhorn/issues/10121) - @derekbit @chriscchien - [IMPROVEMENT] Separate the default backup target settings from default settings [10089](https://github.com/longhorn/longhorn/issues/10089) - @mantissahz @roger-ryao - [IMPROVEMENT] Add support for JSON log format configuration in Longhorn components (UI, driver) [10064](https://github.com/longhorn/longhorn/issues/10064) - @IshinMV @chriscchien - [IMPROVEMENT] No need to start `tgtd` in an instance-manager pod for v2 data engine [9941](https://github.com/longhorn/longhorn/issues/9941) - @derekbit @roger-ryao - [IMPROVEMENT] Configure the log level of other system and user managed components via longhorn manager setting [6702](https://github.com/longhorn/longhorn/issues/6702) - @james-munson @roger-ryao - [IMPROVEMENT] Longhorn CLI should install `cryptsetup` [9315](https://github.com/longhorn/longhorn/issues/9315) - @mantissahz @roger-ryao - [IMPROVEMENT] Collect and display disk space usage for the backing images [8757](https://github.com/longhorn/longhorn/issues/8757) - @ChanYiLin @roger-ryao - [IMPROVEMENT] Check NFS versions in /etc/nfsmount.conf instead [9830](https://github.com/longhorn/longhorn/issues/9830) - @COLDTURNIP @roger-ryao - [IMPROVEMENT] Change misleading error message to warning level [9916](https://github.com/longhorn/longhorn/issues/9916) - @yangchiu @derekbit - [IMPROVEMENT] No need to redirect `spdk_tgt` log to `/var/log/spdk_tgt.log` [9926](https://github.com/longhorn/longhorn/issues/9926) - @derekbit @chriscchien - [UI][IMPROVEMENT] Make backup wait until there is no backup being delete and Add the progress time [8750](https://github.com/longhorn/longhorn/issues/8750) - @a110605 - [IMPROVEMENT] Allow specify data engine version for the default storageclass during Helm installation [9584](https://github.com/longhorn/longhorn/issues/9584) - @shuo-wu @chriscchien - [IMPROVEMENT] Building longhorn-manager takes long time [8744](https://github.com/longhorn/longhorn/issues/8744) - @derekbit @chriscchien - [IMPROVEMENT] Reject strict-local + RWX volume creation [6735](https://github.com/longhorn/longhorn/issues/6735) - @COLDTURNIP @chriscchien - [IMPROVEMENT] Restored volume or other operations should be allowed on the old and running instance-manager pods [9383](https://github.com/longhorn/longhorn/issues/9383) - @derekbit @chriscchien - [IMPROVEMENT] Make backup deletion async and force backup creation wait until there is no backup being delete [8746](https://github.com/longhorn/longhorn/issues/8746) - @ChanYiLin @roger-ryao - [IMPROVEMENT][UI] Attach table search keyword to URL on Backup pages [9974](https://github.com/longhorn/longhorn/issues/9974) - @chriscchien @houhoucoop - [IMPROVEMENT] Busrt ISCSI Connection Errors, and IM Pod Restarting to make LH Volume disconnection [9851](https://github.com/longhorn/longhorn/issues/9851) - @ChanYiLin @chriscchien - [IMPROVEMENT] Why is it not possible to change the replica count in v2 longhorn volume? [9805](https://github.com/longhorn/longhorn/issues/9805) - @hookak @chriscchien - [IMPROVEMENT][UI] Add backupBackingImage table in backup page with tabs [8956](https://github.com/longhorn/longhorn/issues/8956) - @chriscchien @houhoucoop - [IMPROVEMENT] Decrease the reconnect delay of v2 volume nvme initiator [9818](https://github.com/longhorn/longhorn/issues/9818) - @derekbit @chriscchien - [UI][IMPROVEMENT] Improve the volume size information on UI [8843](https://github.com/longhorn/longhorn/issues/8843) - @yangchiu @houhoucoop - [IMPROVEMENT] Add dmsetup and dmcrypt utilities check in cli [8217](https://github.com/longhorn/longhorn/issues/8217) - @COLDTURNIP @roger-ryao - [UI IMPROVEMENT] Collect and display disk space usage for the backing images [9325](https://github.com/longhorn/longhorn/issues/9325) - @chriscchien @houhoucoop - [IMPROVEMENT] Check kernel module `dm_crypt` on host machines [9153](https://github.com/longhorn/longhorn/issues/9153) - @mantissahz @roger-ryao - [IMPROVEMENT] Logging the reason why the instance manager pod is going to be deleted. [9886](https://github.com/longhorn/longhorn/issues/9886) - @derekbit @chriscchien - [IMPROVEMENT] Remove isCLIAPIVersionOne related codes [7191](https://github.com/longhorn/longhorn/issues/7191) - @derekbit @chriscchien - [IMPROVEMENT] Prevent Volume Resize Stuck [6633](https://github.com/longhorn/longhorn/issues/6633) - @c3y1huang @roger-ryao - [IMPROVEMENT] Expose EngineImage CLIAPIVersion and ControllerAPIVersion [6531](https://github.com/longhorn/longhorn/issues/6531) - @chriscchien @houhoucoop - [IMPROVEMENT] Update longhorn-share-manager's nfs-ganesha to v6.2 [9614](https://github.com/longhorn/longhorn/issues/9614) - @derekbit @chriscchien - [UI][IMPROVEMENT] Display `ReUploadedDataSize` and `NewlyUploadedDataSize` of each backup on Backup page [8975](https://github.com/longhorn/longhorn/issues/8975) - @yangchiu @houhoucoop - [IMPROVEMENT] Remove the default CSI component images in longhorn-manager [9580](https://github.com/longhorn/longhorn/issues/9580) - @yangchiu @c3y1huang - [IMPROVEMENT] the `if-not-present` volume backup policy should also check if the latest backup is up-to-date [6027](https://github.com/longhorn/longhorn/issues/6027) - @c3y1huang @chriscchien - [IMPROVEMENT] longhorn/go-common-libs should support Darwin [9487](https://github.com/longhorn/longhorn/issues/9487) - @Yu-Jack @chriscchien - [IMPROVEMENT] Enable Prometheus metrics of the CSI sidecar components [7938](https://github.com/longhorn/longhorn/issues/7938) - @yangchiu @ChanYiLin - [IMPROVEMENT] Talos support for environment check in longhorn manager [9558](https://github.com/longhorn/longhorn/issues/9558) - @yangchiu @c3y1huang - [IMPROVEMENT] Refactor condition icon and text in edit node and disk modal [9238](https://github.com/longhorn/longhorn/issues/9238) - @a110605 - [IMPROVEMENT] Read-only volume monitoring check [8508](https://github.com/longhorn/longhorn/issues/8508) - @ChanYiLin @chriscchien - [IMPROVEMENT] Fix contradicting node status events [7738](https://github.com/longhorn/longhorn/issues/7738) - @yangchiu @ejweber - [IMPROVEMENT] Informative warning message for the failed backup/restore lock acquisition [8713](https://github.com/longhorn/longhorn/issues/8713) - @ChanYiLin @roger-ryao - [IMPROVEMENT] backup backing image should store the secret and secret namespace if it is encrypted [8884](https://github.com/longhorn/longhorn/issues/8884) - @yangchiu @ChanYiLin - [IMPROVEMENT] Start searching for the new available port from the last allocated port instead of starting from 0th port. [8598](https://github.com/longhorn/longhorn/issues/8598) - @james-munson @chriscchien - [IMPROVEMENT] Resilience handling for the last replica timeout [8711](https://github.com/longhorn/longhorn/issues/8711) - @yangchiu @ejweber - [IMPROVEMENT] update `azure-sdk-for-go` version to stable version [8965](https://github.com/longhorn/longhorn/issues/8965) - @mantissahz @chriscchien - [IMPROVEMENT] `toomanysnapshots` UI element not prominent enough to prevent runaway snapshots [6560](https://github.com/longhorn/longhorn/issues/6560) - @a110605 @roger-ryao ### Bug - [BUG] insufficient storage;precheck new replica failed after a temporary shutdown of a node [10210](https://github.com/longhorn/longhorn/issues/10210) - @PhanLe1010 @roger-ryao - [BUG] Checksum of a block device of a v2 volume with multiple replicas might continue to change [10135](https://github.com/longhorn/longhorn/issues/10135) - @derekbit @chriscchien - [BUG] DR volume gets stuck in `Attaching/Detaching` loop when creating a system backup [10239](https://github.com/longhorn/longhorn/issues/10239) - @yangchiu @mantissahz - [BUG] Instability after power failure [9978](https://github.com/longhorn/longhorn/issues/9978) - @yangchiu @james-munson - [BUG] UI Backups list hides the "next 10" buttons [10211](https://github.com/longhorn/longhorn/issues/10211) - @chriscchien @houhoucoop - [BUG] All Backups are lost in the Backup Target if the NFS Service Disconnects and Reconnects again [9530](https://github.com/longhorn/longhorn/issues/9530) - @yangchiu @mantissahz - [BUG] Failed to create system backup if a backup volume with the same name as a volume already exists [10215](https://github.com/longhorn/longhorn/issues/10215) - @mantissahz @roger-ryao - [BUG] v2 engine crash after a rebuilding failure [10212](https://github.com/longhorn/longhorn/issues/10212) - @shuo-wu @chriscchien - [BUG] CSI plugin pod keep crashing util the backup volume appears when creation a backup via the CSI snapshotter [10008](https://github.com/longhorn/longhorn/issues/10008) - @mantissahz @chriscchien @roger-ryao - [BUG] Incorrect field name for backup target in printcolumn [10191](https://github.com/longhorn/longhorn/issues/10191) - @derekbit @mantissahz @roger-ryao - [BUG] BackupTargetController.bsTimerMap should be protected by lock [10190](https://github.com/longhorn/longhorn/issues/10190) - @derekbit @roger-ryao - [BUG] Error backups don't appear on Longhorn UI Backup page [10182](https://github.com/longhorn/longhorn/issues/10182) - @yangchiu @a110605 @mantissahz - [BUG] Fix checksum inconsistent issue after closing the file handler [9876](https://github.com/longhorn/longhorn/issues/9876) - @ChanYiLin @chriscchien - [BUG] Storage doesn't reschedule in v1.7.2 [10090](https://github.com/longhorn/longhorn/issues/10090) - @yangchiu @PhanLe1010 - [BUG][v1.8.0-rc2] Accidentally encountered errors when uninstalling Longhorn [10104](https://github.com/longhorn/longhorn/issues/10104) - @yangchiu @mantissahz - [BUG] Unable to upgrade Longhorn from `v1.7.2` to `v1.8.x-head` if there's an error backup [10181](https://github.com/longhorn/longhorn/issues/10181) - @yangchiu @mantissahz - [BUG][v1.8.x] Can not create backup, backup become in error state immediately [10165](https://github.com/longhorn/longhorn/issues/10165) - @PhanLe1010 @chriscchien - [BUG][v1.8.x] V2 DR volume becomes faulted if the backup deleted before the activation(test_dr_volume_with_backup_and_backup_volume_deleted) [10160](https://github.com/longhorn/longhorn/issues/10160) - @c3y1huang @chriscchien - [BUG] engine.status.snapshots of a DR volume is not correctly updated [10139](https://github.com/longhorn/longhorn/issues/10139) - @derekbit @chriscchien - [BUG] Replica address is missing in the v2 volume backup status response from spdk engine [10136](https://github.com/longhorn/longhorn/issues/10136) - @derekbit @chriscchien - [BUG] V2 volume checksum kept changing [9998](https://github.com/longhorn/longhorn/issues/9998) - @PhanLe1010 @chriscchien - [BUG] v2 volume replica scheduling failed due to insufficient storage but there is available space on the disks [10116](https://github.com/longhorn/longhorn/issues/10116) - @derekbit @chriscchien - [BUG] v2 volume/backup created in v1.7.2 get stuck in attaching/detaching loop after upgrade to v1.8.0-rc2 [10131](https://github.com/longhorn/longhorn/issues/10131) - @yangchiu @derekbit - [BUG][v1.8.0-rc1] Default backup target is periodically cleared [10043](https://github.com/longhorn/longhorn/issues/10043) - @mantissahz @roger-ryao - [BUG] Test case `test_csi_encrypted_block_volume` fails on GKE COS_CONTAINERD [10049](https://github.com/longhorn/longhorn/issues/10049) - @c3y1huang @chriscchien - [BUG] race condition of bdev map get could lead to replica being deleted from the spdk server [9953](https://github.com/longhorn/longhorn/issues/9953) - @derekbit @shuo-wu @roger-ryao - [BUG] Test Case `test_setting` Randomly Fails [10077](https://github.com/longhorn/longhorn/issues/10077) - @mantissahz @roger-ryao - [BUG] Engine Upgrade to 1.7.1 fails on volumes with strict-local data locality [9389](https://github.com/longhorn/longhorn/issues/9389) - @yangchiu @james-munson - [BUG] Default disk registration sets Storage Reserved to 0 instead of using storage-reserved-percentage-for-default-disk value (Engine v2 block type only) [9871](https://github.com/longhorn/longhorn/issues/9871) - @jangseon-ryu @chriscchien - [BUG] Detached Volume Stuck in Attached State During Node Eviction [9781](https://github.com/longhorn/longhorn/issues/9781) - @c3y1huang @chriscchien - [BUG] Data Not Retained After Restoring System Backup [10058](https://github.com/longhorn/longhorn/issues/10058) - @mantissahz @chriscchien @roger-ryao - [BUG] v2 Volume Data Locality `Disabled` Behavior Incorrectly (Switches to `Best-Effort` After Node Drain) [9591](https://github.com/longhorn/longhorn/issues/9591) - @mantissahz @roger-ryao - [BUG] [v1.8.0-rc1] v2 backing image disk download randomly stuck at File processing is not started [10075](https://github.com/longhorn/longhorn/issues/10075) - @ChanYiLin @chriscchien - [BUG] Test case test_system_backup_and_restore_volume_with_backingimage fails [10057](https://github.com/longhorn/longhorn/issues/10057) - @mantissahz @chriscchien - [BUG] longhorn-manager seems to crash rpm-DB on the host by continuously calling rpm -q ... [10019](https://github.com/longhorn/longhorn/issues/10019) - @COLDTURNIP @roger-ryao - [BUG] Backing Image Creation with Multiple Copies Fails in Loop [9976](https://github.com/longhorn/longhorn/issues/9976) - @ChanYiLin @roger-ryao - [BUG] Webhook servers initialization blocks longhorn-manager from running [10054](https://github.com/longhorn/longhorn/issues/10054) - @c3y1huang @chriscchien - [BUG] Missing `fromBackup` Parameter in API Request When Restoring Multiple Files from Backup List [10050](https://github.com/longhorn/longhorn/issues/10050) - @a110605 @roger-ryao - [BUG] Unable to backup backing image through Longhorn UI [10023](https://github.com/longhorn/longhorn/issues/10023) - @a110605 @ChanYiLin @chriscchien - [BUG] Unable to update volume backup target in volume detail page [10045](https://github.com/longhorn/longhorn/issues/10045) - @a110605 @chriscchien - [BUG] Unable to delete backing image backup through UI [10047](https://github.com/longhorn/longhorn/issues/10047) - @chriscchien @houhoucoop - [BUG] Frequent Failures When Uploading 512MB Backing Image [9975](https://github.com/longhorn/longhorn/issues/9975) - @ChanYiLin @roger-ryao - [BUG][v1.8.0-rc1] Error Status When Setting BackupTarget to AWS S3 [10026](https://github.com/longhorn/longhorn/issues/10026) - @mantissahz @roger-ryao - [BUG] Unable to restore latest backup from backup page [10046](https://github.com/longhorn/longhorn/issues/10046) - @mantissahz @roger-ryao - [BUG] Raw format backing image unable to transfer to other nodes to maintain minimum number of copies in Talos v1.8.3 [9882](https://github.com/longhorn/longhorn/issues/9882) - @yangchiu @ChanYiLin - [BUG][v1.8.x] Unable to add block disk after node deleted and added back [10035](https://github.com/longhorn/longhorn/issues/10035) - @yangchiu @derekbit - [BUG] Test case `test_backup_volume_list` failed: failed to find backup in `bv.backupList().data` [9987](https://github.com/longhorn/longhorn/issues/9987) - @yangchiu @mantissahz - [BUG] BackupBackingImage Create failed because it failed to get the BackingImage [10020](https://github.com/longhorn/longhorn/issues/10020) - @ChanYiLin @chriscchien - [BUG] Error creating backup for v2 volume [10013](https://github.com/longhorn/longhorn/issues/10013) - @yangchiu @ChanYiLin - [BUG] Backup does not appear on the backup page [9985](https://github.com/longhorn/longhorn/issues/9985) - @yangchiu @mantissahz - [BUG] volume FailedMount - Input/output error [9939](https://github.com/longhorn/longhorn/issues/9939) - @yangchiu @PhanLe1010 - [BUG] (v2 volume) orphan longhorn device and dm device on the node when IM pod crash [9959](https://github.com/longhorn/longhorn/issues/9959) - @c3y1huang @chriscchien - [BUG] Test case `Stopped replicas on deleted nodes should not be counted as healthy replicas when draining nodes` fails [9616](https://github.com/longhorn/longhorn/issues/9616) - @yangchiu @derekbit - [BUG] Test case test_node_eviction_multiple_volume failed to reschedule replicas after volume detached [9857](https://github.com/longhorn/longhorn/issues/9857) - @yangchiu @c3y1huang - [BUG] Fast-failover lease needs a fail-safe mechanism to clear delinquent state. [9093](https://github.com/longhorn/longhorn/issues/9093) - @yangchiu @james-munson - [BUG] Fail to create v2 volume from VolumeSnapshot with Source:snapshotHandle as Backup [9965](https://github.com/longhorn/longhorn/issues/9965) - @derekbit @chriscchien - [BUG] v2 volume restoration fail if the backup is created by csi-snapshotter [9962](https://github.com/longhorn/longhorn/issues/9962) - @derekbit @chriscchien - [BUG] v2 volume stuck in crash loop due to replica r.ActiveChain is out of sync [9942](https://github.com/longhorn/longhorn/issues/9942) - @shuo-wu @roger-ryao - [BUG] Old backups are not cleaned up after timeout [8319](https://github.com/longhorn/longhorn/issues/8319) - @yangchiu @mantissahz - [BUG] CLI check preflight glosses over absence of NFS installation. [9495](https://github.com/longhorn/longhorn/issues/9495) - @COLDTURNIP @roger-ryao - [BUG] Backport lvol flush implementation from `spdk:v24.01` to `spdk:v24.09` and fix memory leak [8730](https://github.com/longhorn/longhorn/issues/8730) - @yangchiu @DamiaSan - [BUG] v2 volume cannot be attach after upgrading [9943](https://github.com/longhorn/longhorn/issues/9943) - @derekbit @roger-ryao - [BUG] Longhorn is unable to provision volumes larger than 20TB [9221](https://github.com/longhorn/longhorn/issues/9221) - @derekbit @chriscchien @DamiaSan - [BUG] Longhorn v2 volume is stuck in detaching/attaching loop forever if replica crash [9919](https://github.com/longhorn/longhorn/issues/9919) - @derekbit @chriscchien - [BUG] v2 volume may enter ERROR state after deleting an instance manager containing a replica [9874](https://github.com/longhorn/longhorn/issues/9874) - @derekbit @roger-ryao - [BUG] DR volume fails to reattach and faulted after node stop and start during incremental restore [9752](https://github.com/longhorn/longhorn/issues/9752) - @c3y1huang @roger-ryao - [BUG] Backup progress should not add block failed to upload to successful count [9791](https://github.com/longhorn/longhorn/issues/9791) - @derekbit @chriscchien - [BUG] v2 volume faulted during degraded availability restore [9852](https://github.com/longhorn/longhorn/issues/9852) - @c3y1huang @chriscchien - [BUG] Longhorn volume cannot attach to new node permanently if shutdown the current attached node. [9854](https://github.com/longhorn/longhorn/issues/9854) - @yangchiu @PhanLe1010 - [BUG] Replica deletion fails at the end of longhorn-spdk-engine unit test [9121](https://github.com/longhorn/longhorn/issues/9121) - @shuo-wu @chriscchien - [BUG] UnknowOS Message in Longhorn Node Condition on RHEL [9829](https://github.com/longhorn/longhorn/issues/9829) - @yangchiu @mantissahz @jangseon-ryu - [BUG] Failed to inspect the backup backing image information if NFS backup target URL with options [9702](https://github.com/longhorn/longhorn/issues/9702) - @yangchiu @mantissahz - [BUG] Longhorn keeps resetting my storageClass [9391](https://github.com/longhorn/longhorn/issues/9391) - @yangchiu @mantissahz - [BUG] Pre-upgrade pod should event the reason for any failures. [9569](https://github.com/longhorn/longhorn/issues/9569) - @yangchiu @james-munson - [BUG] nvme cli somehow shows the error `malloc(): unsorted double linked list corrupted` [7693](https://github.com/longhorn/longhorn/issues/7693) - @DamiaSan @roger-ryao - [BUG] Host machine is somehow reboot when running into IO timeout [7697](https://github.com/longhorn/longhorn/issues/7697) - @DamiaSan @roger-ryao - [BUG] `spdk_tgt` is somehow crashed [7559](https://github.com/longhorn/longhorn/issues/7559) - @DamiaSan - [BUG] longhornctl install preflight --operating-system=cos failed on COS_CONTAINERD [9664](https://github.com/longhorn/longhorn/issues/9664) - @c3y1huang @chriscchien - [BUG] PV Annotation Isn't Updated After Creating An Oversize Volume [9514](https://github.com/longhorn/longhorn/issues/9514) - @c3y1huang @chriscchien - [BUG] System Restore Stuck at Pending due to Tolerations not Applied [8656](https://github.com/longhorn/longhorn/issues/8656) - @yangchiu @c3y1huang - [BUG] Instance manager missing required selector labels after manager crash [9464](https://github.com/longhorn/longhorn/issues/9464) - @c3y1huang @chriscchien - [BUG] Single Replica Node Down test cases fail [9622](https://github.com/longhorn/longhorn/issues/9622) - @yangchiu @c3y1huang - [BUG] Disks modal broken layout [9629](https://github.com/longhorn/longhorn/issues/9629) - @a110605 @roger-ryao - [BUG] Volume creation violated numberOfReplicas spec causing an unexpected engine image reference count [8573](https://github.com/longhorn/longhorn/issues/8573) - @yangchiu @ChanYiLin - [BUG] kubectl drain node is blocked by unexpected orphan engine processes [6552](https://github.com/longhorn/longhorn/issues/6552) - @yangchiu @PhanLe1010 @ejweber - [BUG] Remove unnecessary restart of RWX workload. [9095](https://github.com/longhorn/longhorn/issues/9095) - @yangchiu @james-munson - [BUG] Fix test case test_rwx_delete_share_manager_pod failure after changes to RWX workload restart. [9504](https://github.com/longhorn/longhorn/issues/9504) - @yangchiu @james-munson - [BUG] longhorn-spdk-engine fail to build [9517](https://github.com/longhorn/longhorn/issues/9517) - @derekbit - [BUG] Volume stuck in degraded [9279](https://github.com/longhorn/longhorn/issues/9279) - @yangchiu @PhanLe1010 - [BUG] Resize RWX PVC Fails [6050](https://github.com/longhorn/longhorn/issues/6050) - @james-munson @roger-ryao - [BUG] Longhorn did not close and open encrypted volumes correctly when the service k3s-agent restarted for a while [9385](https://github.com/longhorn/longhorn/issues/9385) - @yangchiu @mantissahz - [BUG] Faulted RWX volume upon creation [7802](https://github.com/longhorn/longhorn/issues/7802) - @yangchiu @james-munson - [BUG] Accidentally encountered a single replica volume backup stuck at progress 17% indefinitely after a node rebooted [9168](https://github.com/longhorn/longhorn/issues/9168) - @ChanYiLin - [BUG] Longhorn thinks node is unschedulable [9011](https://github.com/longhorn/longhorn/issues/9011) - @c3y1huang @roger-ryao - [BUG] System Backup Fails and DR Volume Enters Attach-Detach Loop When Volume Backup Policy is Set to `Always` [9330](https://github.com/longhorn/longhorn/issues/9330) - @c3y1huang @roger-ryao - [BUG] v1 volume replica rebuild fail after upgrade from v1.7.0 to v1.7.1-rc1 [9331](https://github.com/longhorn/longhorn/issues/9331) - @yangchiu @PhanLe1010 - [BUG] LH fails silently when node has attached volumes [9183](https://github.com/longhorn/longhorn/issues/9183) - @yangchiu @ejweber @w13915984028 - [BUG] Fix security issues in v1.7.1 RC images [9354](https://github.com/longhorn/longhorn/issues/9354) - @c3y1huang - [BUG] Some volumes stuck in "Attaching" state after upgrade to 1.7.0 [9267](https://github.com/longhorn/longhorn/issues/9267) - @ChanYiLin @roger-ryao - [BUG] [Backupstore] Need to close the reader after downloading files for the Azure backup store driver. [9281](https://github.com/longhorn/longhorn/issues/9281) - @yangchiu @mantissahz - [BUG] v2-data-engine setting validator doesn't take disabled nodes into account when checking hugepages [9319](https://github.com/longhorn/longhorn/issues/9319) - @tserong @roger-ryao - [BUG] error logs appeared in uninstallation job [9303](https://github.com/longhorn/longhorn/issues/9303) - @ChanYiLin @chriscchien - [BUG] Incorrect NFS endpoint after enable/disable storage network for RWX volume [9272](https://github.com/longhorn/longhorn/issues/9272) - @Vicente-Cheng @roger-ryao - [BUG][v1.7.x-head] Test case `test_dr_volume_with_backup_block_deletion_abort_during_backup_in_progress` failed due to `failed lock *.lck type 1 acquisition` [9037](https://github.com/longhorn/longhorn/issues/9037) - @ChanYiLin @chriscchien - [BUG] Test case test_support_bundle_should_not_timeout timed out [8507](https://github.com/longhorn/longhorn/issues/8507) - @c3y1huang ### Performance - [TASK] An Investigation for Future Longhorn performance [9552](https://github.com/longhorn/longhorn/issues/9552) - @PhanLe1010 - [TASK] Performance benchmark of v1.7.0 [9350](https://github.com/longhorn/longhorn/issues/9350) - @derekbit ### Misc - [TASK] Fix CVE issues for v1.8.0 [9895](https://github.com/longhorn/longhorn/issues/9895) - @c3y1huang - [TASK] Verify Longhorn v1.8.0 on Talos v1.8.3 [9764](https://github.com/longhorn/longhorn/issues/9764) - @yangchiu @roger-ryao - [DOC] Use experimental for alpha features and preview for beta features [9794](https://github.com/longhorn/longhorn/issues/9794) - @derekbit - [REFACTOR] Consolidate the replica related maps such as bdev map, mode map and address map in SPDK engine [9244](https://github.com/longhorn/longhorn/issues/9244) - @yangchiu @shuo-wu - [IMPROVEMENT] Encrypted volume online expansion support after k8s 1.29 [9902](https://github.com/longhorn/longhorn/issues/9902) - @COLDTURNIP @mantissahz @chriscchien - [DOC] Add an important note about checking items before upgrade [9440](https://github.com/longhorn/longhorn/issues/9440) - @james-munson @chriscchien - [TASK] Bump nfs-ganesha to v6.3 [9979](https://github.com/longhorn/longhorn/issues/9979) - @james-munson @chriscchien - [TASK] No need to add `kind/test` tickets to Longhorn Sprint board [9873](https://github.com/longhorn/longhorn/issues/9873) - @yangchiu - [TASK] Creation of longhorn-v24.09 branch [9700](https://github.com/longhorn/longhorn/issues/9700) - @DamiaSan @roger-ryao - [DOC] Emphasize the backupstore cannot set retention policy [9795](https://github.com/longhorn/longhorn/issues/9795) - @derekbit @roger-ryao - [FEATURE] Inquiry: Storage Reserved Change for Block Type Disks in Engine v2 [9842](https://github.com/longhorn/longhorn/issues/9842) - @derekbit @jangseon-ryu @chriscchien - [TASK] Install the latest grpc_health_probe at build time [9714](https://github.com/longhorn/longhorn/issues/9714) - @yangchiu @c3y1huang - [TASK] Upgrade nvme-cli to v2.10.2 [9739](https://github.com/longhorn/longhorn/issues/9739) - @derekbit - [TASK] Fix CVE issues in support bundle [9658](https://github.com/longhorn/longhorn/issues/9658) - @yangchiu @c3y1huang - [TASK] Update CSI components [9561](https://github.com/longhorn/longhorn/issues/9561) - @c3y1huang - [DOC] Clarify RWX volume expansion documentation [6216](https://github.com/longhorn/longhorn/issues/6216) - @james-munson @roger-ryao - [DOC] Update `Kubernetes Version` in `Best Practices` [9497](https://github.com/longhorn/longhorn/issues/9497) - @derekbit - [TASK] Check Longhorn compatibility with more kind of Linux distros [4900](https://github.com/longhorn/longhorn/issues/4900) - @ChanYiLin @roger-ryao - [TASK] Down size longhorn CLI [8995](https://github.com/longhorn/longhorn/issues/8995) - @c3y1huang ## Contributors - @COLDTURNIP - @ChanYiLin - @DamiaSan - @IshinMV - @PhanLe1010 - @Vicente-Cheng - @Yu-Jack - @a110605 - @c3y1huang - @chriscchien - @derekbit - @ejweber - @hookak - @houhoucoop - @innobead - @james-munson - @jangseon-ryu - @mantissahz - @roger-ryao - @shuo-wu - @tserong - @w13915984028 - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.8.1.md ================================================ ## Longhorn v1.8.1 Release Notes Longhorn 1.8.1 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.8.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.8.1/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.7.x or v1.8.x (< v1.8.1) to v1.8.1.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.8.1/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.8.1][IMPROVEMENT] Support configurable upgrade-responder URL [10439](https://github.com/longhorn/longhorn/issues/10439) - @derekbit @roger-ryao - [BACKPORT][v1.8.1][IMPROVEMENT] Several warning for unknown reason [10420](https://github.com/longhorn/longhorn/issues/10420) - @roger-ryao - [BACKPORT][v1.8.1][IMPROVEMENT] Settings change validation should go back to using Volume state to determine "are all volumes detached" [10376](https://github.com/longhorn/longhorn/issues/10376) - @yangchiu @james-munson ### Bug - [BACKPORT][v1.8.1][BUG] csi keeps creating backup if the backup target is unavailable [10510](https://github.com/longhorn/longhorn/issues/10510) - @mantissahz @roger-ryao - [BACKPORT][v1.8.1][BUG] integer divide by zero in replica scheduler [10506](https://github.com/longhorn/longhorn/issues/10506) - @c3y1huang @chriscchien - [BACKPORT][v1.8.1][BUG] Leading or trailing spaces in Longhorn UI break search [10508](https://github.com/longhorn/longhorn/issues/10508) - @houhoucoop @roger-ryao - [BACKPORT][v1.8.1][BUG] When replica rebuilding completed, the progress could be 99 instead of 100 [10485](https://github.com/longhorn/longhorn/issues/10485) - @shuo-wu @chriscchien - [BACKPORT][v1.8.1][BUG] list_backupVolume API could randomly returns `failed to find a node that is ready and has the default engine image` error [10478](https://github.com/longhorn/longhorn/issues/10478) - @yangchiu @mantissahz - [BACKPORT][v1.8.1][BUG] nil pointer when the backing image copy is delete from the spec but also gets evicted at the same time [10466](https://github.com/longhorn/longhorn/issues/10466) - @yangchiu @ChanYiLin - [BACKPORT][v1.8.1][BUG] 2 uninstall pods could be created after uninstall job was created, one failed with `deleting-confirmation-flag is set to false` error, while the other completed successfully [10484](https://github.com/longhorn/longhorn/issues/10484) - - [BACKPORT][v1.8.1][BUG][UI] Backup store setting doesn't apply to the cloned volume [10468](https://github.com/longhorn/longhorn/issues/10468) - @yangchiu @mantissahz - [BACKPORT][v1.8.1][BUG] v2 volume workload FailedMount with message Staging target path `/var/lib/kubelet/plugins/kubernetes.io/csi/driver.longhorn.io/xxx/globalmount is no longer valid` [10477](https://github.com/longhorn/longhorn/issues/10477) - - [BACKPORT][v1.8.1][BUG][UI] Bulk backup creation with a detached volume returns error 405 and error messages show in browser console [10462](https://github.com/longhorn/longhorn/issues/10462) - @mantissahz - [BACKPORT][v1.8.1][BUG] V2 volume fails to cleanup error replica and rebuild new one - test_data_locality_basic [10364](https://github.com/longhorn/longhorn/issues/10364) - @shuo-wu @chriscchien - [BACKPORT][v1.8.1][BUG] Data lost caused by Longhorn CSI plugin doing a wrong filesystem format action in a rare race condition [10418](https://github.com/longhorn/longhorn/issues/10418) - @yangchiu @PhanLe1010 - [BACKPORT][v1.8.1][BUG] v2 Engine loops in detaching and attaching state after rebuilding [10397](https://github.com/longhorn/longhorn/issues/10397) - @shuo-wu - [BACKPORT][v1.8.1][BUG] A V2 volume checksum will change after replica rebuilding if the volume created with backing image [10341](https://github.com/longhorn/longhorn/issues/10341) - @shuo-wu @chriscchien - [BACKPORT][v1.8.1][BUG] Bug in snapshot count enforcement cause volume faulted and stuck in detaching/attaching loop [10309](https://github.com/longhorn/longhorn/issues/10309) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.8.1][BUG] Test case `test_csi_mount_volume_online_expansion` is failing due to unable to expand PVC [10414](https://github.com/longhorn/longhorn/issues/10414) - @yangchiu @c3y1huang - [BACKPORT][v1.8.1][BUG] V2 BackingImage failed after node reboot [10343](https://github.com/longhorn/longhorn/issues/10343) - @ChanYiLin @chriscchien - [BACKPORT][v1.8.1][BUG] Workload pod will not be able to move to new node when backup operation is taking a long time [10172](https://github.com/longhorn/longhorn/issues/10172) - @PhanLe1010 @chriscchien - [BACKPORT][v1.8.1][BUG] WebUI Volumes Disappear and Reappear [10332](https://github.com/longhorn/longhorn/issues/10332) - @PhanLe1010 @chriscchien @houhoucoop - [BACKPORT][v1.8.1][BUG] "Error get size" from "metrics_collector.(*BackupCollector).Collect" on every metric scrape [10361](https://github.com/longhorn/longhorn/issues/10361) - @derekbit @chriscchien - [BACKPORT][v1.8.1][BUG] [UI] 'Create' button on the System Backup page is disabled after reloading page [10354](https://github.com/longhorn/longhorn/issues/10354) - @chriscchien @houhoucoop - [BACKPORT][v1.8.1][BUG] Proxy gRPC API ReplicaList returns different output formats for v1 and v2 volumes [10353](https://github.com/longhorn/longhorn/issues/10353) - @shuo-wu @roger-ryao - [BACKPORT][v1.8.1][BUG] constant attaching/reattaching of volumes after upgrading to 1.8 [10315](https://github.com/longhorn/longhorn/issues/10315) - @james-munson - [BACKPORT][v1.8.1][BUG] Backup Execution Timeout setting issue in Helm chart [10325](https://github.com/longhorn/longhorn/issues/10325) - @james-munson @chriscchien - [BACKPORT][v1.8.1][BUG] v2 engine stuck in detaching-attaching loop if the previous replica is not cleaned up correct [10363](https://github.com/longhorn/longhorn/issues/10363) - @shuo-wu @chriscchien - [BACKPORT][v1.8.1][BUG] Longhorn CSI plugin 1.8.0 crashes consistently when trying to create a snapshot [10319](https://github.com/longhorn/longhorn/issues/10319) - @PhanLe1010 @chriscchien - [BACKPORT][v1.8.1][BUG] Engine stuck in "stopped" state, prevent volume attach [10329](https://github.com/longhorn/longhorn/issues/10329) - @ChanYiLin @chriscchien - [BACKPORT][v1.8.1][BUG] After upgrading to v1.8.0 the version number lost on the web-ui [10337](https://github.com/longhorn/longhorn/issues/10337) - @derekbit - [BACKPORT][v1.8.1][BUG] insufficient storage;precheck new replica failed after a temporary shutdown of a node [10234](https://github.com/longhorn/longhorn/issues/10234) - @PhanLe1010 ## Misc - [TASK] Fix CVE issues for v1.8.1 [10318](https://github.com/longhorn/longhorn/issues/10318) - @c3y1huang ## Contributors - @ChanYiLin - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @houhoucoop - @innobead - @james-munson - @mantissahz - @roger-ryao - @shuo-wu - @yangchiu - @jillian-maroket - @jhkrug - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.8.2.md ================================================ ## Longhorn v1.8.2 Release Notes Longhorn 1.8.2 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.8.2.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.8.2/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.7.x or v1.8.x (< v1.8.2) to v1.8.2.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.8.2/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.8.2][IMPROVEMENT] Adding retry logic for longhorn-csi-plugin when it is trying to contact the longhorn-manager pods [11027](https://github.com/longhorn/longhorn/issues/11027) - @PhanLe1010 @roger-ryao - [BACKPORT][v1.8.2][IMPROVEMENT] add strict field validation to the update option in upgrade path [10648](https://github.com/longhorn/longhorn/issues/10648) - @ChanYiLin - [BACKPORT][v1.8.2][IMPROVEMENT] Move `SettingNameV2DataEngineHugepageLimit` to danger zone settings [10568](https://github.com/longhorn/longhorn/issues/10568) - @derekbit @chriscchien - [BACKPORT][v1.8.2][IMPROVEMENT] Reduce auto balancing logging noise for detached volumes [10692](https://github.com/longhorn/longhorn/issues/10692) - @roger-ryao - [BACKPORT][v1.8.2][IMPROVEMENT] Improve the Warning Message When Failed to Remove `Block`-Type Disks [10576](https://github.com/longhorn/longhorn/issues/10576) - @yangchiu @ChanYiLin ### Bug - [BACKPORT][v1.8.2][BUG] unable to clean up the backing image volume replica after node eviction [11057](https://github.com/longhorn/longhorn/issues/11057) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.8.2][BUG] backing image volume replica NPE crash during evicting node [11036](https://github.com/longhorn/longhorn/issues/11036) - @COLDTURNIP @chriscchien - [BUG] V2 Backing image failed after upgrade from v1.8.1 to v1.8.2-rc1 [10969](https://github.com/longhorn/longhorn/issues/10969) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.8.2][BUG] Error on git checkout in a container [10975](https://github.com/longhorn/longhorn/issues/10975) - @derekbit @chriscchien - [BACKPORT][v1.8.2][BUG] Helm persistence.backupTargetName not referenced in storageclass template [10964](https://github.com/longhorn/longhorn/issues/10964) - @yangchiu @mantissahz - [BACKPORT][v1.8.2][BUG] MultiUnmapper floods logs with warnings about size mismatch. [10565](https://github.com/longhorn/longhorn/issues/10565) - @shuo-wu @roger-ryao - [BACKPORT][v1.8.2][BUG] Test case `test_snapshot_prune_and_coalesce_simultaneously_with_backing_image` fails [10822](https://github.com/longhorn/longhorn/issues/10822) - @yangchiu @c3y1huang - [BACKPORT][v1.8.2][BUG] System backup could get stuck in `CreatingBackingImageBackups` indefinitely [10748](https://github.com/longhorn/longhorn/issues/10748) - @yangchiu @ChanYiLin - [BACKPORT][v1.8.2][BUG] Failed to terminate namespace `longhorn-system` if there is a support bundle `ReadyForDownload` [10732](https://github.com/longhorn/longhorn/issues/10732) - @yangchiu @c3y1huang - [BACKPORT][v1.8.2][BUG] [v1.9.0-rc1] DR volume does not sync with latest backup when activation [10842](https://github.com/longhorn/longhorn/issues/10842) - @c3y1huang @chriscchien - [BACKPORT][v1.8.2][BUG] Can NOT delete an oversized Not Ready volume [10742](https://github.com/longhorn/longhorn/issues/10742) - @WebberHuang1118 @chriscchien - [BACKPORT][v1.8.2][BUG][UI] Bulk backup creation with a detached volume returns error 405 and error messages show in browser console [10725](https://github.com/longhorn/longhorn/issues/10725) - @yangchiu @a110605 - [BACKPORT][v1.8.2][BUG] Naming collision when creating the name of the new backing image manager [10618](https://github.com/longhorn/longhorn/issues/10618) - @yangchiu @ChanYiLin - [BACKPORT][v1.8.2][BUG] I/O errors on Longhorn v1.7.2 volume during VM migration while upgrading Harvester v1.4.1 [10549](https://github.com/longhorn/longhorn/issues/10549) - @derekbit @roger-ryao - [BACKPORT][v1.8.2][BUG] Adding a non-existing disk to a node will cause the longhorn-manager to crash [10750](https://github.com/longhorn/longhorn/issues/10750) - @ChanYiLin @roger-ryao - [BACKPORT][v1.8.2][BUG] After node down and force delete the terminating deployment pod, volume can not attach success [10713](https://github.com/longhorn/longhorn/issues/10713) - @c3y1huang @chriscchien - [BACKPORT][v1.8.2][BUG] Instance manager image build fail [10654](https://github.com/longhorn/longhorn/issues/10654) - @shuo-wu - [BACKPORT][v1.8.2][BUG] Longhorn Volume Encryption Not Working in Talos 1.9.x [10605](https://github.com/longhorn/longhorn/issues/10605) - @c3y1huang @roger-ryao - [BACKPORT][v1.8.2][BUG] integer divide by zero in replica scheduler [10504](https://github.com/longhorn/longhorn/issues/10504) - @c3y1huang ## Misc - [TASK] Fix CVE issues for v1.8.2 [10913](https://github.com/longhorn/longhorn/issues/10913) - @c3y1huang ## Contributors - @COLDTURNIP - @ChanYiLin - @WebberHuang1118 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @innobead - @mantissahz - @roger-ryao - @shuo-wu - @yangchiu - @sushant-suse - @jillian-maroket - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.9.0.md ================================================ ## Longhorn v1.9.0 Release Notes Longhorn v1.9.0 introduces new features, enhancements, and bug fixes aimed at improving system stability and user experience. Key highlights include V2 Data Engine improvements, orphaned instance deletion, offline replica rebuilding, recurring system backups, and enhanced observability of Longhorn resources. The Longhorn team appreciates your contributions and anticipates receiving feedback regarding this release. For terminology and background on Longhorn releases, see [Releases](https://github.com/longhorn/longhorn#releases). ## Removal ### Environment Check Script The `environment_check.sh` script, deprecated in v1.7.0, has been removed in v1.9.0. Use the [Longhorn Command Line Tool](https://longhorn.io/docs/1.9.0/advanced-resources/longhornctl/) (`longhornctl`) to check your environment for potential issues. ### Orphan Resource Auto-Deletion The `orphan-auto-deletion` setting has been replaced by `orphan-resource-auto-deletion` in v1.9.0. To replicate the previous behavior, include `replica-data` in the `orphan-resource-auto-deletion` value. During the upgrade, the original `orphan-auto-deletion` setting is automatically migrated. For more information, see [Orphan Data Cleanup](https://longhorn.io/docs/1.9.0/advanced-resources/data-cleanup/). ### Deprecated Fields in `longhorn.io/v1beta2` CRDs Deprecated fields have been removed from the CRDs. For details, see issue [#6684](https://github.com/longhorn/longhorn/issues/6684). ## Deprecation & Incompatibilities ### `longhorn.io/v1beta1` API The `v1beta1` version of the Longhorn API is marked unserved and unsupported in v1.9.0 and will be removed in v1.10.0. For more details, see [Issue #10250](https://github.com/longhorn/longhorn/issues/10250). ### Breaking Change in V2 Backing Image Starting with Longhorn v1.9.0, V2 backing images are incompatible with earlier versions due to naming conflicts in the extended attributes (`xattrs`) used by SPDK backing image logical volumes. As a result, V2 backing images must be deleted and recreated during the upgrade process. Since backing images cannot be deleted while volumes using them still exist, you must first back up, delete, and later restore those volumes as the following steps: - Before upgrading to v1.9.0: - Verify that backup targets are functioning properly. - Create full backups of all volumes that use a V2 backing image. - Detach and delete these volumes after the backups complete. - In the **Backing Image** page, save the specifications of all V2 backing images, including the name and the image source. - Delete all V2 backing images. - After upgrading: - Recreate the V2 backing images using the same names and image sources. - Restore the volumes from your backups. For more details, see [Issue #10805](https://github.com/longhorn/longhorn/issues/10805). ## Primary Highlights ### New V2 Data Engine Features While the V2 Data Engine remains experimental in this release, several core functions have been significantly improved: - [Support UBLK Frontend](https://github.com/longhorn/longhorn/issues/9719): Support for UBLK frontend in the V2 Data Engine, which allows for better performance and resource utilization. - [Storage Network](https://github.com/longhorn/longhorn/issues/6450): Introduces support for storage networks in the V2 Data Engine to allow network segregation. - [Offline Replica Rebuilding](https://github.com/longhorn/longhorn/issues/8443): Support for offline replica rebuilding, which allows degraded volumes to automatically recover replicas even while the volume is detached. This capability ensures high data availability without manual intervention. ### Recurring System Backup Starting with Longhorn v1.9.0, you can create a recurring job for system backup creation. [Documentation](https://longhorn.io/docs/1.9.0/advanced-resources/system-backup-restore/backup-longhorn-system/) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/6534) ### Offline Replica Rebuilding Longhorn introduces offline replica rebuilding, a feature that allows degraded volumes to automatically recover replicas even while the volume is detached. This capability minimizes the need for manual recovery steps, accelerates restoration, and ensures high data availability. By default, offline replica rebuilding is disabled. To enable it, set the `offline-replica-rebuilding` setting to `true` in the Longhorn UI or CLI. [Documentation](https://longhorn.io/docs/1.9.0/advanced-resources/rebuilding/offline-replica-rebuilding/) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/8443) ### Orphaned Instance Deletion Longhorn can now track and remove orphaned instances, which are leftover resources like replicas or engines that are no longer associated with an active volume. These instances may accumulate due to unexpected failures or incomplete cleanup. To reduce resource usage and maintain system performance, Longhorn supports both automatic and manual cleanup. By default, this feature is disabled. To enable it, set the `orphan-resource-auto-deletion` setting to `instance` in the Longhorn UI or CLI. [Documentation](https://longhorn.io/docs/1.9.0/advanced-resources/data-cleanup/orphaned-instance-cleanup/) | [GitHub Issue](https://github.com/longhorn/longhorn/issues/6764) ### Improved Metrics for Replica, Engine, and Rebuild Status Longhorn improves observability with new Prometheus metrics that expose the status and identity of Replica and Engine CRs, along with rebuild activity. These metrics make it easier to monitor rebuilds across the cluster. For more information, see [#10550](https://github.com/longhorn/longhorn/issues/10550) and [#10722](https://github.com/longhorn/longhorn/issues/10722). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.9.0.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.9.0/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.8.x to v1.9.0.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.9.0/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ### Highlight - [FEATURE] Cleanup orphaned volume runtime resources if the v1 resources already deleted [6764](https://github.com/longhorn/longhorn/issues/6764) - @COLDTURNIP @chriscchien - [FEATURE] v2 volume supports UBLK frontend [9456](https://github.com/longhorn/longhorn/issues/9456) - @PhanLe1010 @chriscchien - [FEATURE] V1 and V2 volume offline replica rebuilding [8443](https://github.com/longhorn/longhorn/issues/8443) - @mantissahz @roger-ryao - [TASK] Migrate v1beta1 CR to v1beta2 [10250](https://github.com/longhorn/longhorn/issues/10250) - @COLDTURNIP @roger-ryao - [FEATURE] Storage network with V2 data engine [6450](https://github.com/longhorn/longhorn/issues/6450) - @c3y1huang @roger-ryao - [FEATURE] Recurring system backup [6534](https://github.com/longhorn/longhorn/issues/6534) - @yangchiu @c3y1huang ### Feature - [FEATURE] Delta Replica Rebuilding using Delta Snapshot: SPDK API Development [10799](https://github.com/longhorn/longhorn/issues/10799) - @yangchiu @DamiaSan - [FEATURE] Running replicas field in volume table [10817](https://github.com/longhorn/longhorn/issues/10817) - @xelab04 @roger-ryao - [FEATURE] Longhorn UI supports orphaned instance CRs management [10760](https://github.com/longhorn/longhorn/issues/10760) - @yangchiu @houhoucoop - [FEATURE] Allow auto deleting snapshot when a backup is created from that snapshot. [9213](https://github.com/longhorn/longhorn/issues/9213) - @yangchiu @mantissahz - [FEATURE] Add missing metrics of number of volumes/replicas by node/cluster [7599](https://github.com/longhorn/longhorn/issues/7599) - @c3y1huang @roger-ryao ### Improvement - [IMPROVEMENT] Remove unnecessary lasso dependency [10856](https://github.com/longhorn/longhorn/issues/10856) - @derekbit @chriscchien - [IMPROVEMENT] Configurable wait interval for orphaned instance CR creation [10904](https://github.com/longhorn/longhorn/issues/10904) - @derekbit @chriscchien - [IMPROVEMENT] Prevent creating orphans while deleting the v1 instances [10888](https://github.com/longhorn/longhorn/issues/10888) - @COLDTURNIP @chriscchien - [IMPROVEMENT] prevent false logs from deleting volume with offline rebuilding is disabled. [10889](https://github.com/longhorn/longhorn/issues/10889) - @mantissahz @chriscchien - [IMPROVEMENT] Add Prometheus metrics for Replica and Engine CRs [10722](https://github.com/longhorn/longhorn/issues/10722) - @hookak @chriscchien - [IMPROVEMENT] Export longhorn engine rebuild status as prometheus metrics [10550](https://github.com/longhorn/longhorn/issues/10550) - @hookak @chriscchien - [IMPROVEMENT] Disable Snapshot Checksum Calculation for Single-Replica V1 Volume [10518](https://github.com/longhorn/longhorn/issues/10518) - @derekbit @chriscchien - [IMPROVEMENT] add extraObject in charts [10835](https://github.com/longhorn/longhorn/issues/10835) - @DrummyFloyd @chriscchien - [IMPROVEMENT] Disable the v2 snapshot hashing while it is being deleted [10563](https://github.com/longhorn/longhorn/issues/10563) - @shuo-wu @roger-ryao - [IMPROVEMENT] v2 checksum calculation and update should follow the v1 flow [10480](https://github.com/longhorn/longhorn/issues/10480) - @shuo-wu @roger-ryao - [IMPROVEMENT] add strict field validation to the update option in upgrade path [10644](https://github.com/longhorn/longhorn/issues/10644) - @ChanYiLin @chriscchien - [IMPROVEMENT] Show snapshot size during in-progress backup [9783](https://github.com/longhorn/longhorn/issues/9783) - @yangchiu @houhoucoop - [IMPROVEMENT] Don't synchronize all filesystem before snapshotting a v2 volume [9023](https://github.com/longhorn/longhorn/issues/9023) - @yangchiu @DamiaSan - [IMPROVEMENT] spdk_tgt can cancel lvol checksum calculation while there is high priority task [10421](https://github.com/longhorn/longhorn/issues/10421) - @yangchiu @DamiaSan - [IMPROVEMENT] Lvol is not force-removed if Blob is busy [10474](https://github.com/longhorn/longhorn/issues/10474) - @yangchiu @DamiaSan - [IMPROVEMENT] Remove deprecated fields from CRDs [6684](https://github.com/longhorn/longhorn/issues/6684) - @derekbit @roger-ryao - [IMPROVEMENT] Longhorn CLI fails to recognize Raspbian OS [10676](https://github.com/longhorn/longhorn/issues/10676) - @bachmanity1 @roger-ryao - [IMPROVEMENT] Reduce auto balancing logging noise for detached volumes [10691](https://github.com/longhorn/longhorn/issues/10691) - @dihmandrake @roger-ryao - [IMPROVEMENT] Remove the upper bound of v2-data-engine-guaranteed-instance-manager-cpu [10662](https://github.com/longhorn/longhorn/issues/10662) - @derekbit @roger-ryao - [IMPROVEMENT] Clean up BackupTarget condition message handling [8224](https://github.com/longhorn/longhorn/issues/8224) - @chriscchien @houhoucoop - [IMPROVEMENT] Longhorn CLI supports SLES micro [9256](https://github.com/longhorn/longhorn/issues/9256) - @yangchiu @DamiaSan - [IMPROVEMENT] Allow `volumeBindingMode` to be set from helm values [10592](https://github.com/longhorn/longhorn/issues/10592) - @ruant @roger-ryao - [DOC] Prepare a knowledge base for backing image trouble shooting during upgrade [10590](https://github.com/longhorn/longhorn/issues/10590) - @ChanYiLin @chriscchien - [IMPROVEMENT] Missing Prometheus Metrics for Engine v2 Volumes [10472](https://github.com/longhorn/longhorn/issues/10472) - @hookak @roger-ryao - [IMPROVEMENT] Create Volume UI improvement, Automatically Filter Backing Image Based on `v1` or `v2` Selection [10086](https://github.com/longhorn/longhorn/issues/10086) - @houhoucoop @roger-ryao - [UI][IMPROVEMENT] Improve the Warning Message When Failed to Remove `Block`-Type Disks [10580](https://github.com/longhorn/longhorn/issues/10580) - @houhoucoop @roger-ryao - [IMPROVEMENT] Pass full backup mode option to CSI volume snapshot type backup [9785](https://github.com/longhorn/longhorn/issues/9785) - @ChanYiLin @roger-ryao - [UI][IMPROVEMENT] Clean up BackupTarget condition message handling [10579](https://github.com/longhorn/longhorn/issues/10579) - @houhoucoop - [IMPROVEMENT] Improve the Warning Message When Failed to Remove `Block`-Type Disks [10522](https://github.com/longhorn/longhorn/issues/10522) - @yangchiu @ChanYiLin - [IMPROVEMENT] Move `SettingNameV2DataEngineHugepageLimit` to danger zone settings [7746](https://github.com/longhorn/longhorn/issues/7746) - @derekbit @chriscchien - [IMPROVEMENT] Include the /proc/mounts file and multipath.config in the support-bundle [6754](https://github.com/longhorn/longhorn/issues/6754) - @c3y1huang @roger-ryao - [IMPROVEMENT] Use code-generator/kube_codegen.sh to generate K8s stubs and CRDs [7944](https://github.com/longhorn/longhorn/issues/7944) - @derekbit @chriscchien - [IMPROVEMENT] CRD & API code generator decouple from Go conventional source path [10556](https://github.com/longhorn/longhorn/issues/10556) - @COLDTURNIP - [IMPROVEMENT] Support configurable upgrade-responder URL [10437](https://github.com/longhorn/longhorn/issues/10437) - @derekbit @roger-ryao - [IMPROVEMENT] Settings change validation should go back to using Volume state to determine "are all volumes detached" [10233](https://github.com/longhorn/longhorn/issues/10233) - @yangchiu @james-munson - [IMPROVEMENT] Improve the UX of updating danger zone settings [8070](https://github.com/longhorn/longhorn/issues/8070) - @yangchiu @mantissahz - [UI][FEATURE] V1 and V2 volume offline replica rebuilding [10581](https://github.com/longhorn/longhorn/issues/10581) - @houhoucoop @roger-ryao - [UI][FEATURE] Recurring system backup [10262](https://github.com/longhorn/longhorn/issues/10262) - @yangchiu @houhoucoop ### Bug - [BUG] Error on git checkout in a container [10621](https://github.com/longhorn/longhorn/issues/10621) - @derekbit @chriscchien - [BUG] Test case `test_snapshot_prune_and_coalesce_simultaneously_with_backing_image` fails [10808](https://github.com/longhorn/longhorn/issues/10808) - @yangchiu @c3y1huang - [BUG] Failed to terminate namespace `longhorn-system` if there is a support bundle `ReadyForDownload` [10731](https://github.com/longhorn/longhorn/issues/10731) - @yangchiu @c3y1huang - [BUG] Helm persistence.backupTargetName not referenced in storageclass template [10961](https://github.com/longhorn/longhorn/issues/10961) - @yangchiu @mantissahz - [BUG] SPDK API lvol_get_snapshot_range_checksums cannot get the correct result [10950](https://github.com/longhorn/longhorn/issues/10950) - @shuo-wu @roger-ryao - [BUG] V2 Backing image not ready after upgrade from v1.8.1 to v1.9.x [10805](https://github.com/longhorn/longhorn/issues/10805) - @COLDTURNIP @chriscchien - [BUG] v2 replica rebuilding will miss the backing image [10909](https://github.com/longhorn/longhorn/issues/10909) - @shuo-wu @chriscchien - [BUG] minor spacing issues found with yamllint in the helm chart [10681](https://github.com/longhorn/longhorn/issues/10681) - @codekow - [BUG][UI] Snapshots of v2 volume with backing image aren't shown on the `Snapshots and Backups` graph [10526](https://github.com/longhorn/longhorn/issues/10526) - @derekbit @chriscchien - [BUG] Deleted orphan data still renders on the page until page refresh [10803](https://github.com/longhorn/longhorn/issues/10803) - @COLDTURNIP @chriscchien @houhoucoop - [BUG] DR volume does not sync with latest backup when activation [10824](https://github.com/longhorn/longhorn/issues/10824) - @c3y1huang @chriscchien - [BUG][v1.9.0-rc1] Block disks become temporarily unavailable after the upgrading from `v1.8.1` to `v1.9.0-rc1` [10821](https://github.com/longhorn/longhorn/issues/10821) - @mantissahz - [BUG] Naming collision when creating the name of the new backing image manager [10616](https://github.com/longhorn/longhorn/issues/10616) - @yangchiu @ChanYiLin - [BUG] v2 volume replica status `error` after snapshot deletion with `Immediate Data Integrity Check` Enabled [10798](https://github.com/longhorn/longhorn/issues/10798) - @shuo-wu - [BUG] Enabling `V2 Data Engine` setting, v2 instance manager doesn't start after certain negative factor operations [10791](https://github.com/longhorn/longhorn/issues/10791) - @COLDTURNIP @yangchiu - [BUG] spdk emits `Device or resource busy` while registering lvol checksum calculation [10140](https://github.com/longhorn/longhorn/issues/10140) - @shuo-wu @roger-ryao - [BUG] v2 instance managers keep crashing on master-head arm64 environment [10768](https://github.com/longhorn/longhorn/issues/10768) - @yangchiu @PhanLe1010 - [BUG] Wrong image name in `longhorn-images.txt` [10774](https://github.com/longhorn/longhorn/issues/10774) - @c3y1huang - [BUG] After node down and force delete the terminating deployment pod, volume can not attach success [10689](https://github.com/longhorn/longhorn/issues/10689) - @c3y1huang @chriscchien - [BUG] Deleting a replica of one v2 volume will also degrade the other v2 volume [10527](https://github.com/longhorn/longhorn/issues/10527) - @yangchiu @ChanYiLin - [BUG] Adding a non-existing disk to a node will cause the longhorn-manager to crash [10749](https://github.com/longhorn/longhorn/issues/10749) - @ChanYiLin @roger-ryao - [BUG] Upgrading Longhorn from v1.8.1 to master-head causes longhorn-manager to crash [10762](https://github.com/longhorn/longhorn/issues/10762) - @yangchiu @mantissahz - [BUG] After node rebooted and workload pod restarted, pod data size became 0, and the mounted volume turned read-only [9248](https://github.com/longhorn/longhorn/issues/9248) - @yangchiu @c3y1huang - [BUG] Test case `test_engine_crash_during_live_upgrade` failed due to data loss [10751](https://github.com/longhorn/longhorn/issues/10751) - @c3y1huang @roger-ryao - [BUG] System backup could get stuck in `CreatingBackingImageBackups` indefinitely [10740](https://github.com/longhorn/longhorn/issues/10740) - @yangchiu @ChanYiLin - [BUG] v2 volume with backing image gets stuck in `Attaching` state [10743](https://github.com/longhorn/longhorn/issues/10743) - @yangchiu @ChanYiLin - [BUG] Can NOT delete an oversized Not Ready volume [10741](https://github.com/longhorn/longhorn/issues/10741) - @WebberHuang1118 @chriscchien - [BUG][UI] Bulk backup creation with a detached volume returns error 405 and error messages show in browser console [10460](https://github.com/longhorn/longhorn/issues/10460) - @yangchiu @a110605 - [BUG] `spdk_tgt` encountered `Lvol store removed with error: -16` in `longhorn-spdk-helper` during a CI test [10622](https://github.com/longhorn/longhorn/issues/10622) - @derekbit @roger-ryao - [BUG] 2 uninstall pods could be created after uninstall job was created, one failed with `deleting-confirmation-flag is set to false` error, while the other completed successfully [10483](https://github.com/longhorn/longhorn/issues/10483) - @yangchiu @derekbit - [BUG] SPDK constantly emits "Bad length of checksum xattr" [10399](https://github.com/longhorn/longhorn/issues/10399) - @ChanYiLin @chriscchien - [BUG] Can't create v2 block-type disk via BDF on Talos [10313](https://github.com/longhorn/longhorn/issues/10313) - @derekbit @roger-ryao - [BUG] I/O errors on Longhorn v1.7.2 volume during VM migration while upgrading Harvester v1.4.1 [10495](https://github.com/longhorn/longhorn/issues/10495) - @derekbit @roger-ryao - [BUG] MultiUnmapper floods logs with warnings about size mismatch. [6406](https://github.com/longhorn/longhorn/issues/6406) - @shuo-wu @roger-ryao - [BUG] `spdk_tgt` segfaulted in `longhorn-spdk-helper` during a CI test run. [10598](https://github.com/longhorn/longhorn/issues/10598) - @derekbit @roger-ryao - [BUG] Backup Execution Timeout setting issue in Helm chart [10323](https://github.com/longhorn/longhorn/issues/10323) - @yangchiu @james-munson - [BUG] Longhorn Volume Encryption Not Working in Talos 1.9.x [10584](https://github.com/longhorn/longhorn/issues/10584) - @c3y1huang @roger-ryao - [BUG][UI] Inconsistent capitalization in the `Allow snapshots removal during trim` volume setting [10470](https://github.com/longhorn/longhorn/issues/10470) - @yangchiu @houhoucoop - [BUG] Expand Volume option is greyed under Volume tab but working in the volume detail section. [7529](https://github.com/longhorn/longhorn/issues/7529) - @yangchiu @houhoucoop - [BUG] Instance manager image build fail [10653](https://github.com/longhorn/longhorn/issues/10653) - @shuo-wu - [BUG] Recurring job pod stuck in pending state and unable to create new snapshots after node reboot [7956](https://github.com/longhorn/longhorn/issues/7956) - @c3y1huang @chriscchien - [BUG] Instance manager pod stuck in Terminating state after upgrade with v2 backing image [10520](https://github.com/longhorn/longhorn/issues/10520) - @ChanYiLin @chriscchien - [BUG] Extra replica created when create volume in a engine image not fully deployed environment [8263](https://github.com/longhorn/longhorn/issues/8263) - @c3y1huang @chriscchien - [BUG] [v1.8.0-rc1] Uninstallation fail if having backing images, the instance-manager pod stuck at terminating [10044](https://github.com/longhorn/longhorn/issues/10044) - @ChanYiLin @chriscchien - [BUG] test_statefulset_restore fails on integration test run with NFS backup store [3451](https://github.com/longhorn/longhorn/issues/3451) - @roger-ryao - [BUG] csi keeps creating backup if the backup target is unavailable [10501](https://github.com/longhorn/longhorn/issues/10501) - @mantissahz @roger-ryao - [BUG] nil pointer when the backing image copy is delete from the spec but also gets evicted at the same time [10464](https://github.com/longhorn/longhorn/issues/10464) - @yangchiu @ChanYiLin - [BUG] Mutex is copied in getLatestBackup [6965](https://github.com/longhorn/longhorn/issues/6965) - @james-munson @roger-ryao - [BUG] integer divide by zero in replica scheduler [10502](https://github.com/longhorn/longhorn/issues/10502) - @c3y1huang @chriscchien - [BUG] Leading or trailing spaces in Longhorn UI break search [10491](https://github.com/longhorn/longhorn/issues/10491) - @houhoucoop @roger-ryao - [BUG] When replica rebuilding completed, the progress could be 99 instead of 100 [8589](https://github.com/longhorn/longhorn/issues/8589) - @shuo-wu @chriscchien - [BUG] Workload with RWX volume cannot recover when Kubelet restarts [2933](https://github.com/longhorn/longhorn/issues/2933) - @james-munson @chriscchien - [BUG][UI] Backup store setting doesn't apply to the cloned volume [10463](https://github.com/longhorn/longhorn/issues/10463) - @yangchiu @mantissahz - [BUG] Data lost caused by Longhorn CSI plugin doing a wrong filesystem format action in a rare race condition [10416](https://github.com/longhorn/longhorn/issues/10416) - @yangchiu @PhanLe1010 - [BUG] WebUI Volumes Disappear and Reappear [10314](https://github.com/longhorn/longhorn/issues/10314) - @yangchiu @PhanLe1010 @houhoucoop - [BUG] Longhorn-manager logs "Failed to sync backup status" on every backup [10301](https://github.com/longhorn/longhorn/issues/10301) - @derekbit @chriscchien - [BUG] Rebuilding stuck for DR volume if the node was power down while restoring [2747](https://github.com/longhorn/longhorn/issues/2747) - @COLDTURNIP @roger-ryao - [BUG] Uninstalling K3s without uninstalling Longhorn first when Longhorn SPDK volumes exist hangs on arm64 [8132](https://github.com/longhorn/longhorn/issues/8132) - @roger-ryao - [BUG] A V2 volume checksum will change after replica rebuilding if the volume created with backing image [10340](https://github.com/longhorn/longhorn/issues/10340) - @shuo-wu @chriscchien - [BUG] RWX volume becomes faulted after the node reconnects [5658](https://github.com/longhorn/longhorn/issues/5658) - @james-munson @chriscchien - [BUG] V2 BackingImage failed after node reboot [10342](https://github.com/longhorn/longhorn/issues/10342) - @ChanYiLin @chriscchien - [BUG] degraded v2 volume doesn't create new replica even though there is an available disk [9197](https://github.com/longhorn/longhorn/issues/9197) - @c3y1huang - [BUG] Bug in snapshot count enforcement cause volume faulted and stuck in detaching/attaching loop [10308](https://github.com/longhorn/longhorn/issues/10308) - @PhanLe1010 @roger-ryao - [BUG] Test case `test_csi_mount_volume_online_expansion` is failing due to unable to expand PVC [10411](https://github.com/longhorn/longhorn/issues/10411) - @yangchiu @c3y1huang - [BUG] Longhorn CSI plugin 1.8.0 crashes consistently when trying to create a snapshot [10303](https://github.com/longhorn/longhorn/issues/10303) - @yangchiu @PhanLe1010 - [BUG] Workload pod will not be able to move to new node when backup operation is taking a long time [10171](https://github.com/longhorn/longhorn/issues/10171) - @yangchiu @PhanLe1010 - [BUG] v2 engine stuck in detaching-attaching loop if the previous replica is not cleaned up correct [10293](https://github.com/longhorn/longhorn/issues/10293) - @yangchiu @shuo-wu - [BUG] [UI] 'Create' button on the System Backup page is disabled after reloading page [10351](https://github.com/longhorn/longhorn/issues/10351) - @yangchiu @houhoucoop - [BUG] "Error get size" from "metrics_collector.(*BackupCollector).Collect" on every metric scrape [10358](https://github.com/longhorn/longhorn/issues/10358) - @derekbit @chriscchien - [BUG] Proxy gRPC API ReplicaList returns different output formats for v1 and v2 volumes [10347](https://github.com/longhorn/longhorn/issues/10347) - @shuo-wu @roger-ryao - [BUG] Engine stuck in "stopped" state, prevent volume attach [9938](https://github.com/longhorn/longhorn/issues/9938) - @ChanYiLin @roger-ryao - [BUG] After upgrading to v1.8.0 the version number lost on the web-ui [10336](https://github.com/longhorn/longhorn/issues/10336) - @derekbit - [BUG] constant attaching/reattaching of volumes after upgrading to 1.8 [10304](https://github.com/longhorn/longhorn/issues/10304) - @PhanLe1010 - [BUG] Sometimes attached DR volume checksum fluctuates [9305](https://github.com/longhorn/longhorn/issues/9305) - @c3y1huang - [BUG] Backing image manager pods unable to come up on RHEL 8.4 [2767](https://github.com/longhorn/longhorn/issues/2767) - @roger-ryao - [BUG] Failed to upgrade Longhorn from `v1.8.x-head` to `master-head` [10143](https://github.com/longhorn/longhorn/issues/10143) - @roger-ryao - [BUG] [v1.5.4-rc2] V2 volume perform engine upgrade when concurrent-automatic-engine-upgrade-per-node-limit > 0 [7930](https://github.com/longhorn/longhorn/issues/7930) - @derekbit - [BUG] Negative test case got stuck in waiting for longhorn-ui pods [8248](https://github.com/longhorn/longhorn/issues/8248) - @c3y1huang ### Misc - [DOC] Replica count behaviour is unclear. [10861](https://github.com/longhorn/longhorn/issues/10861) - @hoo29 @chriscchien - [DOC] Update "Upgrade Path Enforcement and Downgrade Prevention" [10945](https://github.com/longhorn/longhorn/issues/10945) - @derekbit @roger-ryao - [TASK] Update the longhornio/nfs-ganesha image [10878](https://github.com/longhorn/longhorn/issues/10878) - @derekbit @c3y1huang @chriscchien - [DOC] No `BackupTargetSecret` in `Settings/General` [10858](https://github.com/longhorn/longhorn/issues/10858) - @vnwnv @roger-ryao - [DOC] Create system backup first before upgrade system [10633](https://github.com/longhorn/longhorn/issues/10633) - @ChanYiLin @chriscchien - [TASK] [UI] [FEATURE] v2 volume supports UBLK frontend [10735](https://github.com/longhorn/longhorn/issues/10735) - @chriscchien @houhoucoop - [DOC] Adding support for RKE2/k3s in data-recovery steps [10714](https://github.com/longhorn/longhorn/issues/10714) - @mattmattox @roger-ryao - [DOC] Architecture Diagram [6761](https://github.com/longhorn/longhorn/issues/6761) - @derekbit @chriscchien - [TASK] Fix longhorn/website to support latest Hugo server version [10632](https://github.com/longhorn/longhorn/issues/10632) - @chriscchien @sushant-suse - [TASK] fix lint problems in longhorn-manager [10639](https://github.com/longhorn/longhorn/issues/10639) - @COLDTURNIP @chriscchien - [DOC] Codeblocks in KB don't line wrap [8143](https://github.com/longhorn/longhorn/issues/8143) - @roger-ryao @sushant-suse - [TASK] Longhorn UI assessment for ui-extension migration [10487](https://github.com/longhorn/longhorn/issues/10487) - @houhoucoop - [DOC] Update the steps to set up the Azure backup target [9688](https://github.com/longhorn/longhorn/issues/9688) - @mantissahz - [DOC] Explain the process of creating backing images from existing volumes [10093](https://github.com/longhorn/longhorn/issues/10093) - @ChanYiLin @chriscchien - [DOC] Update ArgoCD installation document [10588](https://github.com/longhorn/longhorn/issues/10588) - @mantissahz - [TASK] Remove environment check script since v1.9.0 [9239](https://github.com/longhorn/longhorn/issues/9239) - @yangchiu @derekbit - [TASK] Add platform architect and volume encryption info to metrics [7047](https://github.com/longhorn/longhorn/issues/7047) - @c3y1huang @roger-ryao - [DOC] Update the document and create a KB to address the limitation that BackingImage should be multiple of 512B [10536](https://github.com/longhorn/longhorn/issues/10536) - @ChanYiLin - [Doc] Clarifications on defaultSettings.defaultDataLocality and persistence.defaultDataLocality usage [10253](https://github.com/longhorn/longhorn/issues/10253) - @james-munson @roger-ryao ## New Contributors - @bachmanity1 - @codekow - @DrummyFloyd - @hoo29 - @hookak - @dihmandrake - @mattmattox - @ruant - @vnwnv - @xelab04 ## Contributors - @COLDTURNIP - @ChanYiLin - @DamiaSan - @PhanLe1010 - @WebberHuang1118 - @a110605 - @c3y1huang - @chriscchien - @derekbit - @houhoucoop - @innobead - @james-munson - @mantissahz - @roger-ryao - @shuo-wu - @yangchiu - @sushant-suse - @jillian-maroket - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.9.1.md ================================================ ## Longhorn v1.9.1 Release Notes Longhorn 1.9.1 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.9.1.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.9.1/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.8.x or v1.9.x (< v1.9.1) to v1.9.1.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.9.1/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Feature - [BACKPORT][v1.9.1][FEATURE] Standardized way to override container image registry [11068](https://github.com/longhorn/longhorn/issues/11068) - @marcosbc @roger-ryao - [BACKPORT][v1.9.1][FEATURE] Standardized way to specify image pull secrets [11072](https://github.com/longhorn/longhorn/issues/11072) - @marcosbc @chriscchien ### Improvement - [BACKPORT][v1.9.1][IMPROVEMENT] Remove the Patch `preserveUnknownFields: false` for CRDs [11280](https://github.com/longhorn/longhorn/issues/11280) - @derekbit @chriscchien - [BACKPORT][v1.9.1][IMPROVEMENT] Improve the disk space un-schedulable condition message [11212](https://github.com/longhorn/longhorn/issues/11212) - @yangchiu @davidcheng0922 - [BACKPORT][v1.9.1][IMPROVEMENT] Improve the condition message of engine image check [11196](https://github.com/longhorn/longhorn/issues/11196) - @derekbit @chriscchien - [BACKPORT][v1.9.1][IMPROVEMENT] Improve the logging when detecting multiple backup volumes of the same volume on the same backup target [11225](https://github.com/longhorn/longhorn/issues/11225) - @PhanLe1010 @chriscchien - [BACKPORT][v1.9.1][IMPROVEMENT] extra invalid BackupVolumeCR may be created during cluster split-brain [11168](https://github.com/longhorn/longhorn/issues/11168) - @mantissahz @roger-ryao - [BACKPORT][v1.9.1][IMPROVEMENT] Full replica rebuilding when a node goes down for a while and then comes back [11069](https://github.com/longhorn/longhorn/issues/11069) - @mantissahz - [BACKPORT][v1.9.1][IMPROVEMENT] Adding retry logic for longhorn-csi-plugin when it trying to contact the longhorn-manager pods [10914](https://github.com/longhorn/longhorn/issues/10914) - @PhanLe1010 @roger-ryao ### Bug - [BACKPORT][v1.9.1][BUG] Incorrect value of `remove-snapshots-during-filesystem-trim` in longhorn chart/values.yaml [11266](https://github.com/longhorn/longhorn/issues/11266) - @derekbit @chriscchien - [BACKPORT][v1.9.1][BUG] privateRegistry.registryUrl does not work when overriding specific image registries [11258](https://github.com/longhorn/longhorn/issues/11258) - @marcosbc @chriscchien - [BACKPORT][v1.9.1][BUG] system backup error [11235](https://github.com/longhorn/longhorn/issues/11235) - @c3y1huang @roger-ryao - [BACKPORT][v1.9.1][BUG] Volume expansion fails with "unsupported disk encryption format ext4" [11184](https://github.com/longhorn/longhorn/issues/11184) - @COLDTURNIP @mantissahz @roger-ryao - [BACKPORT][v1.9.1][BUG] The Backup YAML example in the Longhorn doc does not work [11217](https://github.com/longhorn/longhorn/issues/11217) - @mantissahz @nzhan126 - [BACKPORT][v1.9.1][BUG] CSI Plugin restart triggers unintended restart of migratable RWX volume workloads [11164](https://github.com/longhorn/longhorn/issues/11164) - @c3y1huang @roger-ryao - [BACKPORT][v1.9.1][BUG] in the browser UI: Volume -> Clone Volume results in the broken browser page [11180](https://github.com/longhorn/longhorn/issues/11180) - @houhoucoop @roger-ryao - [BACKPORT][v1.9.1][BUG] Test case `test_engine_image_not_fully_deployed_perform_volume_operations` failed: unable to detach a volume [10917](https://github.com/longhorn/longhorn/issues/10917) - @mantissahz @chriscchien - [BACKPORT][v1.9.1][BUG] Creating support-bundle panic NPE [11170](https://github.com/longhorn/longhorn/issues/11170) - @c3y1huang @roger-ryao - [BACKPORT][v1.9.1][BUG] Unable to Build Longhorn-Share-Manager Image Due to CMAKE Compatibility [11162](https://github.com/longhorn/longhorn/issues/11162) - @derekbit @roger-ryao - [BACKPORT][v1.9.1][BUG] Recurring jobs fail when assigned to default group [11020](https://github.com/longhorn/longhorn/issues/11020) - @c3y1huang @chriscchien - [BACKPORT][v1.9.1][BUG] SPDK API bdev_lvol_detach_parent does not work as expected [11047](https://github.com/longhorn/longhorn/issues/11047) - @DamiaSan @roger-ryao - [BACKPORT][v1.9.1][BUG] unable to clean up the backing image volume replica after node eviction [11056](https://github.com/longhorn/longhorn/issues/11056) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.9.1][BUG] backing image volume replica NPE crash during evicting node [11035](https://github.com/longhorn/longhorn/issues/11035) - @COLDTURNIP @chriscchien ## Misc - [HOTFIX] Create hotfixed image for longhorn-manager:v1.9.0 [11140](https://github.com/longhorn/longhorn/issues/11140) - @derekbit @chriscchien - [BACKPORT][v1.9.1][TASK] Ensure support-bundle-kit builds use vendored dependencies [11118](https://github.com/longhorn/longhorn/issues/11118) - @yangchiu @c3y1huang ## New Contributors - @davidcheng0922 - @marcosbc - @nzhan126 ## Contributors - @COLDTURNIP - @DamiaSan - @PhanLe1010 - @c3y1huang - @chriscchien - @derekbit - @houhoucoop - @innobead - @mantissahz - @roger-ryao - @yangchiu - @sushant-suse - @rebeccazzzz - @forbesguthrie - @asettle ================================================ FILE: CHANGELOG/CHANGELOG-1.9.2.md ================================================ ## Longhorn v1.9.2 Release Notes Longhorn 1.9.2 introduces several improvements and bug fixes that are intended to improve system quality, resilience, stability and security. The Longhorn team appreciates your contributions and expects to receive feedback regarding this release. > [!NOTE] > For more information about release-related terminology, see [Releases](https://github.com/longhorn/longhorn#releases). ## Installation > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before installing Longhorn v1.9.2.** You can install Longhorn using a variety of tools, including Rancher, Kubectl, and Helm. For more information about installation methods and requirements, see [Quick Installation](https://longhorn.io/docs/1.9.2/deploy/install/) in the Longhorn documentation. ## Upgrade > [!IMPORTANT] **Ensure that your cluster is running Kubernetes v1.25 or later before upgrading from Longhorn v1.8.x or v1.9.x (< v1.9.2) to v1.9.2.** Longhorn only allows upgrades from supported versions. For more information about upgrade paths and procedures, see [Upgrade](https://longhorn.io/docs/1.9.2/deploy/upgrade/) in the Longhorn documentation. ## Post-Release Known Issues For information about issues identified after this release, see [Release-Known-Issues](https://github.com/longhorn/longhorn/wiki/Release-Known-Issues). ## Resolved Issues ### Improvement - [BACKPORT][v1.9.2][IMPROVEMENT] Add usage metrics for Longhorn installation variant [11805](https://github.com/longhorn/longhorn/issues/11805) - @derekbit - [BACKPORT][v1.9.2][IMPROVEMENT] SAST Potential dereference of the null pointer in controller/volume_controller.go in longhorn-manager [11782](https://github.com/longhorn/longhorn/issues/11782) - @c3y1huang - [BACKPORT][v1.9.2][IMPROVEMENT] Collect mount table, process status and process table in support bundle [11726](https://github.com/longhorn/longhorn/issues/11726) - @mantissahz @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] rename the backing image manager to reduce the probability of CR name collision [11567](https://github.com/longhorn/longhorn/issues/11567) - @COLDTURNIP @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] Improve log messages of longhorn-engine, tgt and liblonghorn for troubleshooting [11604](https://github.com/longhorn/longhorn/issues/11604) - @yangchiu @derekbit - [BACKPORT][v1.9.2][IMPROVEMENT] Misleading log message `Deleting orphans on evicted node ...` [11501](https://github.com/longhorn/longhorn/issues/11501) - @yangchiu @derekbit - [BACKPORT][v1.9.2][IMPROVEMENT] Check if the backup target is available before creating a backup, backup backing image, and system backup [11324](https://github.com/longhorn/longhorn/issues/11324) - @yangchiu @nzhan126 - [BACKPORT][v1.9.2][IMPROVEMENT] adjust the hardcoded timeout limitation for backing image downloading [11310](https://github.com/longhorn/longhorn/issues/11310) - @COLDTURNIP @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] Improve longhorn-engine controller log messages [11508](https://github.com/longhorn/longhorn/issues/11508) - @derekbit @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] Make liveness probe parameters of instance-manager pod configurable [11506](https://github.com/longhorn/longhorn/issues/11506) - @derekbit @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] backing image handle node disk deleting events [11488](https://github.com/longhorn/longhorn/issues/11488) - @COLDTURNIP @chriscchien - [BACKPORT][v1.9.2][IMPROVEMENT] Handle credential secret containing mixed invalid conditions [11327](https://github.com/longhorn/longhorn/issues/11327) - @yangchiu @nzhan126 - [BACKPORT][v1.9.2][IMPROVEMENT] Improve the condition message of engine image check [11193](https://github.com/longhorn/longhorn/issues/11193) - @derekbit @chriscchien ### Bug - [BACKPORT][v1.9.2][BUG] Potential Data Corruption During Volume Resizing When Created from Snapshot [11788](https://github.com/longhorn/longhorn/issues/11788) - @yangchiu @PhanLe1010 - [BUG] [v1.9.x] support bundle stuck at 33% [11744](https://github.com/longhorn/longhorn/issues/11744) - @mantissahz @chriscchien - [BACKPORT][v1.9.2][BUG] Unable to disable v2-data-engine even though there is no v2 volumes, backing images or orphaned data [11639](https://github.com/longhorn/longhorn/issues/11639) - @shuo-wu @chriscchien - [BACKPORT][v1.9.2][BUG] Longhorn pvcs are in pending state. [11722](https://github.com/longhorn/longhorn/issues/11722) - @yangchiu @derekbit - [BUG] Broken link in documentation [11729](https://github.com/longhorn/longhorn/issues/11729) - @consideRatio - [BACKPORT][v1.9.2][BUG] longhornctl preflight install should load and check iscsi_tcp kernel module. [11710](https://github.com/longhorn/longhorn/issues/11710) - @mantissahz @chriscchien - [BACKPORT][v1.9.2][BUG] Backing image download gets stuck after network disconnection [11624](https://github.com/longhorn/longhorn/issues/11624) - @COLDTURNIP - [BACKPORT][v1.9.2][BUG] Volume becomes faulted when its replica node disks run out of space during a write operation [11341](https://github.com/longhorn/longhorn/issues/11341) - @mantissahz @chriscchien - [BACKPORT][v1.9.2][BUG] Engine process continues running after rapid volume detachment [11606](https://github.com/longhorn/longhorn/issues/11606) - @COLDTURNIP @yangchiu @chriscchien - [BACKPORT][v1.9.2][BUG] Creating a 2 Gi volume with a 200 Mi backing image is rejected with “volume size should be larger than the backing image size” [11648](https://github.com/longhorn/longhorn/issues/11648) - @COLDTURNIP @yangchiu @chriscchien - [BACKPORT][v1.9.2][BUG] longhorn-manager repeatedly emits `No instance manager for node xxx for update instance state of orphan instance orphan-xxx..` [11599](https://github.com/longhorn/longhorn/issues/11599) - @COLDTURNIP @chriscchien - [BACKPORT][v1.9.2][BUG] BackupBackingImage may be created from an unready BackingImageManager [11692](https://github.com/longhorn/longhorn/issues/11692) - @WebberHuang1118 @roger-ryao - [BACKPORT][v1.9.2][BUG] Longhorn fails to create Backing Image Backup on ARM platform [11570](https://github.com/longhorn/longhorn/issues/11570) - @COLDTURNIP - [BACKPORT][v1.9.2][BUG] remaining unknown OS condition in node CR [11614](https://github.com/longhorn/longhorn/issues/11614) - @COLDTURNIP @roger-ryao - [BACKPORT][v1.9.2][BUG] Volumes fails to remount when they go read-only [11584](https://github.com/longhorn/longhorn/issues/11584) - @derekbit @chriscchien - [BACKPORT][v1.9.2][BUG] Dangling Volume State When Live Migration Terminates Unexpectedly [11590](https://github.com/longhorn/longhorn/issues/11590) - @PhanLe1010 @chriscchien - [BACKPORT][v1.9.2][BUG] Unable to setup backup target in storage network environment: cannot find a running instance manager for node [11482](https://github.com/longhorn/longhorn/issues/11482) - @derekbit @chriscchien - [BACKPORT][v1.9.2][BUG] Test case `test_recurring_jobs_when_volume_detached_unexpectedly` failed: backup completed but progress did not reach 100% [11476](https://github.com/longhorn/longhorn/issues/11476) - @yangchiu @mantissahz - [BACKPORT][v1.9.2][BUG] Recurring Job with 'default' group causes goroutine deadlock on v1.9.1 (Regression of #11020) [11494](https://github.com/longhorn/longhorn/issues/11494) - @c3y1huang - [BACKPORT][v1.9.2][BUG] Test Case `test_replica_auto_balance_node_least_effort` Is Sometimes Failed [11391](https://github.com/longhorn/longhorn/issues/11391) - @derekbit @chriscchien - [BACKPORT][v1.9.2][BUG] Unable to set up S3 backup target if backups already exist [11344](https://github.com/longhorn/longhorn/issues/11344) - @mantissahz @chriscchien - [BACKPORT][v1.9.2][BUG] longhorn-manager is crashed due to `SIGSEGV: segmentation violation` [11422](https://github.com/longhorn/longhorn/issues/11422) - @derekbit @roger-ryao - [BACKPORT][v1.9.2][BUG] Typo in configuration parameter: "offlineRelicaRebuilding" should be "offlineReplicaRebuilding" [11382](https://github.com/longhorn/longhorn/issues/11382) - @yangchiu - [BUG][UI][v1.9.2-rc2] Unable to Retrieve Volume's Backup List in the Operation [11841](https://github.com/longhorn/longhorn/issues/11841) - @houhouhoucoop @roger-ryao ## New Contributors - @consideRatio ## Contributors - @COLDTURNIP - @PhanLe1010 - @WebberHuang1118 - @c3y1huang - @chriscchien - @derekbit - @innobead - @mantissahz - @nzhan126 - @roger-ryao - @shuo-wu - @yangchiu ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Longhorn Community Code of Conduct Longhorn follows the [Cloud Native Computing Foundation Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Guideline Welcome contributing to Longhorn! This guideline applies to all the repositories under Longhorn. Contributing to Longhorn is not limited to writing the code or submitting the PR. We will also appreciate if you can file issues, provide feedback and suggest new features. In fact, many of Longhorn's features are driven by the community's need. The community plays a big role in the development of Longhorn. Of course, contributing the code is more than welcome. To make things simpler, if you're fixing a small issue (e.g. typo), go ahead submitting a PR and we will pick it up; but if you're planning to submit a bigger PR to implement a new feature, it's easier to submit a new issue to discuss the design with the maintainers first before implementing it. When you're ready to get involved in contributing the code, [this developer guide](https://github.com/longhorn/longhorn/wiki/Getting-started-with-Longhorn-Development) should help you to get up to the speed. And remember to [sign off your commits](#dco-sign-off)! Feel free to join the discussion on Longhorn development at [longhorn-dev](https://rancher-users.slack.com/messages/CMLPKMYDC) slack channel. Happy contributing! ## DCO Sign off All authors to the project retain copyright to their work. However, to ensure that they are only submitting work that they have rights to, we are requiring everyone to acknowledge this by signing their work. Any copyright notices in this repo should specify the authors as "the Longhorn contributors". To sign your work, just add a line like this at the end of your commit message: ``` Signed-off-by: Sheng Yang ``` This can easily be done with the `--signoff/-s` option to `git commit`. By doing this you state that you can certify the following (from https://developercertificate.org/): ``` Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 1 Letterman Drive Suite D4700 San Francisco, CA, 94129 Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MAINTAINERS ================================================ The list of current Longhorn maintainers: Name, , @GitHubHandle, Type Sheng Yang, , @yasker, Dev Shuo Wu, , @shuo-wu, Dev David Ko, , @innobead, Dev Derek Su, , @derekbit, Dev Phan Le, , @PhanLe1010, Dev Chinya Huang, , @c3y1huang, Dev Divya Mohan, , @divya-mohan0209, Community Manager ================================================ FILE: README.md ================================================

Longhorn

A CNCF Incubating Project. Visit longhorn.io for the full documentation.

[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/longhorn/longhorn/badge)](https://scorecard.dev/viewer/?uri=github.com/longhorn/longhorn) [![Releases](https://img.shields.io/github/release/longhorn/longhorn/all.svg)](https://github.com/longhorn/longhorn/releases) [![GitHub](https://img.shields.io/github/license/longhorn/longhorn)](https://github.com/longhorn/longhorn/blob/master/LICENSE) [![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://longhorn.io/docs/latest/)
Longhorn is a distributed block storage system for Kubernetes. Longhorn is cloud-native storage built using Kubernetes and container primitives. Longhorn is lightweight, reliable, and powerful. You can install Longhorn on an existing Kubernetes cluster with one `kubectl apply` command or by using Helm charts. Once Longhorn is installed, it adds persistent volume support to the Kubernetes cluster. Longhorn implements distributed block storage using containers and microservices. Longhorn creates a dedicated storage controller for each block device volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. The storage controller and replicas are themselves orchestrated using Kubernetes. Here are some notable features of Longhorn: 1. Enterprise-grade distributed storage with no single point of failure 2. Incremental snapshot of block storage 3. Backup to secondary storage (NFSv4 or S3-compatible object storage) built on efficient change block detection 4. Recurring snapshot and backup 5. Automated non-disruptive upgrade. You can upgrade the entire Longhorn software stack without disrupting running volumes! 6. Intuitive GUI dashboard You can read more technical details of Longhorn [here](https://longhorn.io/). # Releases > **NOTE**: > - __\*__ means the release branch is under active support and will have periodic follow-up patch releases. > - __Latest__ release means the version is the latest release of the newest release branch. > - __Stable__ release means the version is stable and has been widely adopted by users. > - Release EOL: One year after the first stable version. For the details, please refer to [Release Support](https://github.com/longhorn/longhorn/wiki/Release-Schedule-&-Support#release-support). https://github.com/longhorn/longhorn/releases | Release | Latest Version | Stable Versions | Release Note | Important Note | Active | |-----------|-----------------|-----------------------------------|-----------------------------------------------------------------|--------------------------------------------------------------|--------| | **1.11*** | 1.11.1 | | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.11.1) | [🔗](https://longhorn.io/docs/1.11.1/important-notes) | ✅ | | **1.10*** | 1.10.2 | 1.10.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.10.2) | [🔗](https://longhorn.io/docs/1.10.2/important-notes) | ✅ | | **1.9*** | 1.9.2 | 1.9.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.9.2) | [🔗](https://longhorn.io/docs/1.9.2/important-notes) | ✅ | | **1.8*** | 1.8.2 | 1.8.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.8.2) | [🔗](https://longhorn.io/docs/1.8.2/important-notes) | ✅ | | 1.7 | 1.7.3 | 1.7.3, 1.7.2, 1.7.1 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.7.3) | [🔗](https://longhorn.io/docs/1.7.3/important-notes) | | | 1.6 | 1.6.4 | 1.6.4, 1.6.3, 1.6.2, 1.6.1 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.6.4) | [🔗](https://longhorn.io/docs/1.6.4/deploy/important-notes) | | | 1.5 | 1.5.5 | 1.5.5, 1.5.4, 1.5.3 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.5.5) | [🔗](https://longhorn.io/docs/1.5.5/deploy/important-notes) | | | 1.4 | 1.4.4 | 1.4.4, 1.4.3, 1.4.2, 1.4.1 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.4.4) | [🔗](https://longhorn.io/docs/1.4.4/deploy/important-notes) | | | 1.3 | 1.3.3 | 1.3.3, 1.3.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.3.3) | [🔗](https://longhorn.io/docs/1.3.3/deploy/important-notes) | | | 1.2 | 1.2.6 | 1.2.6, 1.2.5, 1.2.4, 1.2.3, 1.2.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.2.6) | [🔗](https://longhorn.io/docs/1.2.6/deploy/important-notes) | | | 1.1 | 1.1.3 | 1.1.3, 1.1.2 | [🔗](https://github.com/longhorn/longhorn/releases/tag/v1.1.3) | | | # Roadmap https://github.com/longhorn/longhorn/wiki/Roadmap # Components Longhorn is 100% open-source software. Project source code is spread across several repositories: * Manager: [![Build Status](https://github.com/longhorn/longhorn-manager/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/longhorn-manager/actions/workflows/build.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/longhorn/longhorn-manager)](https://goreportcard.com/report/github.com/longhorn/longhorn-manager)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-manager.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-manager?ref=badge_shield&issueType=license) * Instance Manager: [![Build Status](https://github.com/longhorn/longhorn-instance-manager/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/longhorn-instance-manager/actions/workflows/build.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/longhorn/longhorn-instance-manager)](https://goreportcard.com/report/github.com/longhorn/longhorn-instance-manager)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-instance-manager.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-instance-manager?ref=badge_shield&issueType=license) * Engine: [![Build Status](https://github.com/longhorn/longhorn-engine/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/longhorn-engine/actions/workflows/build.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/longhorn/longhorn-engine)](https://goreportcard.com/report/github.com/longhorn/longhorn-engine)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-engine.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-engine?ref=badge_shield&issueType=license) * Share Manager: [![Build Status](https://github.com/longhorn/longhorn-share-manager/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/longhorn-share-manager/actions/workflows/build.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/longhorn/longhorn-share-manager)](https://goreportcard.com/report/github.com/longhorn/longhorn-share-manager)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-share-manager.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-share-manager?ref=badge_shield&issueType=license) * Backing Image Manager: [![Build Status](https://github.com/longhorn/backing-image-manager/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/backing-image-manager/actions/workflows/build.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/longhorn/backing-image-manager)](https://goreportcard.com/report/github.com/longhorn/backing-image-manager)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-backing-image-manager.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-backing-image-manager?ref=badge_shield&issueType=license) * UI: [![Build Status](https://github.com/longhorn/longhorn-ui/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/longhorn-ui/actions/workflows/build.yml)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-ui.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-ui?ref=badge_shield&issueType=license) * CLI: [![build](https://github.com/longhorn/cli/actions/workflows/build.yml/badge.svg)](https://github.com/longhorn/cli/actions/workflows/build.yml)[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B162%2Flonghorn-cli.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B162%2Flonghorn-cli?ref=badge_shield&issueType=license) | Component | What it does | GitHub repo | | :----------------------------- | :--------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | | Longhorn Backing Image Manager | Backing image download, sync, and deletion in a disk | [longhorn/backing-image-manager](https://github.com/longhorn/backing-image-manager) | | Longhorn Instance Manager | Controller/replica instance lifecycle management | [longhorn/longhorn-instance-manager](https://github.com/longhorn/longhorn-instance-manager) | | Longhorn Manager | Longhorn orchestration, includes CSI driver for Kubernetes | [longhorn/longhorn-manager](https://github.com/longhorn/longhorn-manager) | | Longhorn Share Manager | NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes | [longhorn/longhorn-share-manager](https://github.com/longhorn/longhorn-share-manager) | | Longhorn UI | The Longhorn dashboard | [longhorn/longhorn-ui](https://github.com/longhorn/longhorn-ui) | | Longhorn CLI | Command Line Interface for Longhorn | [longhorn/cli](https://github.com/longhorn/cli) | | Library | What it does | GitHub repo | | :----------------------------- | :--------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | | Longhorn Engine | V1 Core controller/replica logic | [longhorn/longhorn-engine](https://github.com/longhorn/longhorn-engine) | | Longhorn SPDK Engine | V2 Core controller/replica logic | [longhorn/longhorn-spdk-engine](https://github.com/longhorn/longhorn-spdk-engine) | | iSCSI Helper | V1 iSCSI client and server libraries | [longhorn/go-iscsi-helper](https://github.com/longhorn/go-iscsi-helper) | | SPDK Helper | V2 SPDK client and server libraries | [longhorn/go-spdk-helper](https://github.com/longhorn/go-spdk-helper) | | Backup Store | Backup libraries | [longhorn/backupstore](https://github.com/longhorn/backupstore) | | Common Libraries | | [longhorn/go-common-libs](https://github.com/longhorn/go-common-libs) | ![Longhorn UI](./longhorn-ui.png) # Get Started ## Requirements For the installation requirements, refer to the [Longhorn documentation.](https://longhorn.io/docs/latest/deploy/install/#installation-requirements) ## Installation > **NOTE**: > Please note that the master branch is for the upcoming feature release development. > For an official release installation or upgrade, please take a look at the ways below. Longhorn can be installed on a Kubernetes cluster in several ways: - [Rancher App Marketplace](https://longhorn.io/docs/latest/deploy/install/install-with-rancher/) - [kubectl](https://longhorn.io/docs/latest/deploy/install/install-with-kubectl/) - [Helm](https://longhorn.io/docs/latest/deploy/install/install-with-helm/) ## Documentation The official Longhorn documentation is [here.](https://longhorn.io/docs) # Get Involved ## Discussion, Feedback If having any discussions or feedback, feel free to [file a discussion](https://github.com/longhorn/longhorn/discussions). ## Features Request, Bug Reporting If having any issues, feel free to [file an issue](https://github.com/longhorn/longhorn/issues/new/choose). We have a weekly community issue review meeting to review all reported issues or enhancement requests. When creating a bug issue, please help upload the support bundle to the issue or send it to [longhorn-support-bundle](mailto:longhorn-support-bundle@suse.com). ## Report Vulnerabilities If any vulnerabilities are found, please report them to [longhorn-security](mailto:longhorn-security@suse.com). # Community Longhorn is open-source software, so contributions are greatly welcome. Please read [Code of Conduct](./CODE_OF_CONDUCT.md) and [Contributing Guideline](./CONTRIBUTING.md) before contributing. Contributing code is not the only way of contributing. We value feedback very much and many of the Longhorn features originated from users' feedback. If you have any feedback, feel free to [file an issue](https://github.com/longhorn/longhorn/issues/new/choose). ## Slack You can also provide feedback or join the conversation with other developers, users, and contributors on the [CNCF](https://slack.cncf.io/) [#longhorn](https://cloud-native.slack.com/messages/longhorn) Slack channel. This is a good place to learn about Longhorn, ask questions, and share your experiences. ## Community Meeting and Office Hours We host a monthly community meeting on the **3rd Thursday**, alternating between *AMER/EU-friendly* and *APAC-friendly times* - at **4 PM UTC** and **8 AM UTC** respectively. Everyone is welcome to join us. You can find the calendar invite [here](https://zoom-lfx.platform.linuxfoundation.org/meetings/longhorn?view=list) ## Longhorn Mailing List Subscribe to our [developer](https://lists.cncf.io/g/cncf-longhorn-dev) and [users](https://lists.cncf.io/g/cncf-longhorn-users) to stay updated with the latest news and events. You can read more about our community and its events here: https://github.com/longhorn/community # License Copyright (c) 2014-2025 The Longhorn Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ## Longhorn is a [CNCF Incubating Project](https://www.cncf.io/projects/) ![Longhorn is a CNCF Incubating Project](https://github.com/cncf/artwork/blob/main/other/cncf/horizontal/color/cncf-color.png) ================================================ FILE: SECURITY.md ================================================ ## Report Vulnerabilities If any vulnerabilities are found, please report them to [longhorn-security](mailto:longhorn-security@suse.com). ================================================ FILE: chart/.helmignore ================================================ # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. .DS_Store # Common VCS dirs .git/ .gitignore .bzr/ .bzrignore .hg/ .hgignore .svn/ # Common backup files *.swp *.bak *.tmp *~ # Various IDEs .project .idea/ *.tmproj ================================================ FILE: chart/Chart.yaml ================================================ apiVersion: v1 name: longhorn version: 1.9.0-dev appVersion: v1.9.0-dev kubeVersion: ">=1.25.0-0" description: Longhorn is a distributed block storage system for Kubernetes. keywords: - longhorn - storage - distributed - block - device - iscsi - nfs home: https://github.com/longhorn/longhorn sources: - https://github.com/longhorn/longhorn - https://github.com/longhorn/longhorn-engine - https://github.com/longhorn/longhorn-instance-manager - https://github.com/longhorn/longhorn-share-manager - https://github.com/longhorn/longhorn-manager - https://github.com/longhorn/longhorn-ui - https://github.com/longhorn/longhorn-tests - https://github.com/longhorn/backing-image-manager maintainers: - name: Longhorn maintainers email: maintainers@longhorn.io icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/longhorn/icon/color/longhorn-icon-color.png ================================================ FILE: chart/README.md ================================================ # Longhorn Chart > **Important**: Please install the Longhorn chart in the `longhorn-system` namespace only. > **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. > **Note**: Use Helm 3 when installing and upgrading Longhorn. Helm 2 is [no longer supported](https://helm.sh/blog/helm-2-becomes-unsupported/). ## Source Code Longhorn is 100% open source software. Project source code is spread across a number of repos: 1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine 2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager 3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager 4. Backing Image Manager -- Backing image file lifecycle management. https://github.com/longhorn/backing-image-manager 5. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager 6. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui ## Prerequisites 1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) 2. Kubernetes >= v1.25 3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. 4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. ## Upgrading to Kubernetes v1.25+ Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. > **Note:** > If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** > > If your charts get stuck in this state, you may have to clean up your Helm release secrets. Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Longhorn docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. ## Installation 1. Add Longhorn chart repository. ``` helm repo add longhorn https://charts.longhorn.io ``` 2. Update local Longhorn chart information from chart repository. ``` helm repo update ``` 3. Use the following commands to create the `longhorn-system` namespace first, then install the Longhorn chart. ``` kubectl create namespace longhorn-system helm install longhorn longhorn/longhorn --namespace longhorn-system ``` ## Uninstallation ``` kubectl -n longhorn-system patch -p '{"value": "true"}' --type=merge lhs deleting-confirmation-flag helm uninstall longhorn -n longhorn-system kubectl delete namespace longhorn-system ``` ## Values The `values.yaml` contains items used to tweak a deployment of this chart. ### Cattle Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | global.cattle.systemDefaultRegistry | string | `""` | Default system registry. | | global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector | string | `"kubernetes.io/os:linux"` | Node selector for system-managed Longhorn components. | | global.cattle.windowsCluster.defaultSetting.taintToleration | string | `"cattle.io/os=linux:NoSchedule"` | Toleration for system-managed Longhorn components. | | global.cattle.windowsCluster.enabled | bool | `false` | Setting that allows Longhorn to run on a Rancher Windows cluster. | | global.cattle.windowsCluster.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for Linux nodes that can run user-deployed Longhorn components. | | global.cattle.windowsCluster.tolerations | list | `[{"effect":"NoSchedule","key":"cattle.io/os","operator":"Equal","value":"linux"}]` | Toleration for Linux nodes that can run user-deployed Longhorn components. | | global.imagePullSecrets | list | `[]` | Global override for image pull secrets for container registry. | | global.imageRegistry | string | `"docker.io"` | Global override for container image registry. | | global.nodeSelector | object | `{}` | Node selector for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer. | | global.timezone | string | `""` | Set container timezone (TZ env) for all Longhorn workloads. Leave empty to use container default. | | global.tolerations | list | `[]` | Toleration for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer. | ### Network Policies | Key | Type | Default | Description | |-----|------|---------|-------------| | networkPolicies.enabled | bool | `false` | Setting that allows you to enable network policies that control access to Longhorn pods. | | networkPolicies.type | string | `"k3s"` | Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") | ### Image Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | image.csi.attacher.registry | string | `""` | Registry for the CSI attacher image. When unspecified, Longhorn uses the default value. | | image.csi.attacher.repository | string | `"longhornio/csi-attacher"` | Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. | | image.csi.attacher.tag | string | `"v4.10.0-20251226"` | Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. | | image.csi.livenessProbe.registry | string | `""` | Registry for the CSI liveness probe image. When unspecified, Longhorn uses the default value. | | image.csi.livenessProbe.repository | string | `"longhornio/livenessprobe"` | Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. | | image.csi.livenessProbe.tag | string | `"v2.17.0-20251226"` | Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. | | image.csi.nodeDriverRegistrar.registry | string | `""` | Registry for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. | | image.csi.nodeDriverRegistrar.repository | string | `"longhornio/csi-node-driver-registrar"` | Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. | | image.csi.nodeDriverRegistrar.tag | string | `"v2.15.0-20251226"` | Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. | | image.csi.provisioner.registry | string | `""` | Registry for the CSI Provisioner image. When unspecified, Longhorn uses the default value. | | image.csi.provisioner.repository | string | `"longhornio/csi-provisioner"` | Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. | | image.csi.provisioner.tag | string | `"v5.3.0-20251226"` | Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. | | image.csi.resizer.registry | string | `""` | Registry for the CSI Resizer image. When unspecified, Longhorn uses the default value. | | image.csi.resizer.repository | string | `"longhornio/csi-resizer"` | Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. | | image.csi.resizer.tag | string | `"v2.0.0-20251226"` | Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. | | image.csi.snapshotter.registry | string | `""` | Registry for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. | | image.csi.snapshotter.repository | string | `"longhornio/csi-snapshotter"` | Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. | | image.csi.snapshotter.tag | string | `"v8.4.0-20251226"` | Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. | | image.longhorn.backingImageManager.registry | string | `""` | Registry for the Backing Image Manager image. When unspecified, Longhorn uses the default value. | | image.longhorn.backingImageManager.repository | string | `"longhornio/backing-image-manager"` | Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. | | image.longhorn.backingImageManager.tag | string | `"master-head"` | Tag for the Backing Image Manager image. When unspecified, Longhorn uses the default value. | | image.longhorn.engine.registry | string | `""` | Registry for the Longhorn Engine image. | | image.longhorn.engine.repository | string | `"longhornio/longhorn-engine"` | Repository for the Longhorn Engine image. | | image.longhorn.engine.tag | string | `"master-head"` | Tag for the Longhorn Engine image. | | image.longhorn.instanceManager.registry | string | `""` | Registry for the Longhorn Instance Manager image. | | image.longhorn.instanceManager.repository | string | `"longhornio/longhorn-instance-manager"` | Repository for the Longhorn Instance Manager image. | | image.longhorn.instanceManager.tag | string | `"master-head"` | Tag for the Longhorn Instance Manager image. | | image.longhorn.manager.registry | string | `""` | Registry for the Longhorn Manager image. | | image.longhorn.manager.repository | string | `"longhornio/longhorn-manager"` | Repository for the Longhorn Manager image. | | image.longhorn.manager.tag | string | `"master-head"` | Tag for the Longhorn Manager image. | | image.longhorn.shareManager.registry | string | `""` | Registry for the Longhorn Share Manager image. | | image.longhorn.shareManager.repository | string | `"longhornio/longhorn-share-manager"` | Repository for the Longhorn Share Manager image. | | image.longhorn.shareManager.tag | string | `"master-head"` | Tag for the Longhorn Share Manager image. | | image.longhorn.supportBundleKit.registry | string | `""` | Registry for the Longhorn Support Bundle Manager image. | | image.longhorn.supportBundleKit.repository | string | `"longhornio/support-bundle-kit"` | Repository for the Longhorn Support Bundle Manager image. | | image.longhorn.supportBundleKit.tag | string | `"v0.0.79"` | Tag for the Longhorn Support Bundle Manager image. | | image.longhorn.ui.registry | string | `""` | Registry for the Longhorn UI image. | | image.longhorn.ui.repository | string | `"longhornio/longhorn-ui"` | Repository for the Longhorn UI image. | | image.longhorn.ui.tag | string | `"master-head"` | Tag for the Longhorn UI image. | | image.openshift.oauthProxy.registry | string | `""` | Registry for the OAuth Proxy image. Specify the upstream image (for example, "quay.io/openshift/origin-oauth-proxy"). This setting applies only to OpenShift users. | | image.openshift.oauthProxy.repository | string | `""` | Repository for the OAuth Proxy image. Specify the upstream image (for example, "quay.io/openshift/origin-oauth-proxy"). This setting applies only to OpenShift users. | | image.openshift.oauthProxy.tag | string | `""` | Tag for the OAuth Proxy image. Specify OCP/OKD version 4.1 or later (including version 4.18, which is available at quay.io/openshift/origin-oauth-proxy:4.18). This setting applies only to OpenShift users. | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI. | ### Service Settings | Key | Description | |-----|-------------| | service.manager.nodePort | NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767. | | service.manager.type | Service type for Longhorn Manager. | | service.ui.annotations | Annotation for the Longhorn UI service. | | service.ui.labels | | | service.ui.loadBalancerClass | Class of a load balancer implementation | | service.ui.nodePort | NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. | | service.ui.type | Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") | ### StorageClass Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | persistence.backingImage.dataSourceParameters | string | `nil` | Data source parameters of a backing image used in a Longhorn StorageClass. You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) | | persistence.backingImage.dataSourceType | string | `nil` | Data source type of a backing image used in a Longhorn StorageClass. If the backing image exists in the cluster, Longhorn uses this setting to verify the image. If the backing image does not exist, Longhorn creates one using the specified data source type. | | persistence.backingImage.enable | bool | `false` | Setting that allows you to use a backing image in a Longhorn StorageClass. | | persistence.backingImage.expectedChecksum | string | `nil` | Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. | | persistence.backingImage.name | string | `nil` | Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. | | persistence.backupTargetName | string | `"default"` | Setting that allows you to specify the backup target for the default Longhorn StorageClass. | | persistence.dataEngine | string | `"v1"` | Setting that allows you to specify the data engine version for the default Longhorn StorageClass. (Options: "v1", "v2") | | persistence.defaultClass | bool | `true` | Setting that allows you to specify the default Longhorn StorageClass. | | persistence.defaultClassReplicaCount | int | `3` | Replica count of the default Longhorn StorageClass. | | persistence.defaultDataLocality | string | `"disabled"` | Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") | | persistence.defaultDiskSelector.enable | bool | `false` | Setting that allows you to enable the disk selector for the default Longhorn StorageClass. | | persistence.defaultDiskSelector.selector | string | `""` | Disk selector for the default Longhorn StorageClass. Longhorn uses only disks with the specified tags for storing volume data. (Examples: "nvme,sata") | | persistence.defaultFsType | string | `"ext4"` | Filesystem type of the default Longhorn StorageClass. | | persistence.defaultMkfsParams | string | `""` | mkfs parameters of the default Longhorn StorageClass. | | persistence.defaultNodeSelector.enable | bool | `false` | Setting that allows you to enable the node selector for the default Longhorn StorageClass. | | persistence.defaultNodeSelector.selector | string | `""` | Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") | | persistence.disableRevisionCounter | string | `"true"` | Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the volume-head-xxx.img file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. | | persistence.migratable | bool | `false` | Setting that allows you to enable live migration of a Longhorn volume from one node to another. | | persistence.nfsOptions | string | `""` | Set NFS mount options for Longhorn StorageClass for RWX volumes | | persistence.reclaimPolicy | string | `"Delete"` | Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") | | persistence.recurringJobSelector.enable | bool | `false` | Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. | | persistence.recurringJobSelector.jobList | list | `[]` | Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) | | persistence.unmapMarkSnapChainRemoved | string | `"ignored"` | Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") | | persistence.volumeBindingMode | string | `"Immediate"` | VolumeBindingMode controls when volume binding and dynamic provisioning should occur. (Options: "Immediate", "WaitForFirstConsumer") (Defaults to "Immediate") | ### CSI Settings | Key | Description | |-----|-------------| | csi.attacherReplicaCount | Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). | | csi.kubeletRootDir | kubelet root directory. When unspecified, Longhorn uses the default value. | | csi.podAntiAffinityPreset | Configures Pod anti-affinity to prevent multiple instances on the same node. Use soft (tries to separate) or hard (must separate). When unspecified, Longhorn uses the default value ("soft"). | | csi.provisionerReplicaCount | Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). | | csi.resizerReplicaCount | Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). | | csi.snapshotterReplicaCount | Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). | ### Longhorn Manager Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn Manager. | Key | Type | Default | Description | |-----|------|---------|-------------| | longhornManager.log.format | string | `"plain"` | Format of Longhorn Manager logs. (Options: "plain", "json") | | longhornManager.nodeSelector | object | `{}` | Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager. | | longhornManager.priorityClass | string | `"longhorn-critical"` | PriorityClass for Longhorn Manager. | | longhornManager.resources | string | `nil` | Resource requests and limits for Longhorn Manager pods. | | longhornManager.serviceAnnotations | object | `{}` | Annotation for the Longhorn Manager service. | | longhornManager.serviceLabels | object | `{}` | | | longhornManager.tolerations | list | `[]` | Toleration for Longhorn Manager on nodes allowed to run Longhorn components. | | longhornManager.updateStrategy.rollingUpdate.maxUnavailable | string | `"100%"` | | ### Longhorn Driver Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn Driver. | Key | Type | Default | Description | |-----|------|---------|-------------| | longhornDriver.log.format | string | `"plain"` | Format of longhorn-driver logs. (Options: "plain", "json") | | longhornDriver.nodeSelector | object | `{}` | Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver. | | longhornDriver.priorityClass | string | `"longhorn-critical"` | PriorityClass for Longhorn Driver. | | longhornDriver.tolerations | list | `[]` | Toleration for Longhorn Driver on nodes allowed to run Longhorn components. | ### Longhorn UI Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn UI. | Key | Type | Default | Description | |-----|------|---------|-------------| | longhornUI.affinity | object | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["longhorn-ui"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}}` | Affinity for Longhorn UI pods. Specify the affinity you want to use for Longhorn UI. | | longhornUI.nodeSelector | object | `{}` | Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI. | | longhornUI.priorityClass | string | `"longhorn-critical"` | PriorityClass for Longhorn UI. | | longhornUI.replicas | int | `2` | Replica count for Longhorn UI. | | longhornUI.tolerations | list | `[]` | Toleration for Longhorn UI on nodes allowed to run Longhorn components. | ### Ingress Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | ingress.annotations | string | `nil` | Ingress annotations in the form of key-value pairs. | | ingress.enabled | bool | `false` | Setting that allows Longhorn to generate ingress records for the Longhorn UI service. | | ingress.extraHosts | list | `[]` | Extra hostnames for TLS (Subject Alternative Names - SAN). Used when you need multiple FQDNs for the same ingress. Example: extraHosts: - longhorn.example.com - longhorn-ui.internal.local | | ingress.host | string | `"sslip.io"` | Hostname of the Layer 7 load balancer. | | ingress.ingressClassName | string | `nil` | IngressClass resource that contains ingress configuration, including the name of the Ingress controller. ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases. | | ingress.path | string | `"/"` | Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}. | | ingress.pathType | string | `"ImplementationSpecific"` | Ingress path type. To maintain backward compatibility, the default value is "ImplementationSpecific". | | ingress.secrets | string | `nil` | Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses. | | ingress.secureBackends | bool | `false` | Setting that allows you to enable secure connections to the Longhorn UI service via port 443. | | ingress.tls | bool | `false` | Setting that allows you to enable TLS on ingress records. | | ingress.tlsSecret | string | `"longhorn.local-tls"` | TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records. | ### HTTPRoute Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | httproute.annotations | object | `{}` | Annotations for the HTTPRoute resource in the form of key-value pairs. | | httproute.enabled | bool | `false` | Setting that allows Longhorn to generate HTTPRoute records for the Longhorn UI service using Gateway API. | | httproute.hostnames | list | `[]` | List of hostnames for the HTTPRoute. Multiple hostnames are supported. | | httproute.parentRefs | list | `[]` | Gateway references for HTTPRoute. Specify which Gateway(s) should handle this route. | | httproute.path | string | `"/"` | Default path for HTTPRoute. You can access the Longhorn UI by following the full path. | | httproute.pathType | string | `"PathPrefix"` | Path match type for HTTPRoute. (Options: "Exact", "PathPrefix") | ### Private Registry Settings You can install Longhorn in an air-gapped environment with a private registry. For more information, see the **Air Gap Installation** section of the [documentation](https://longhorn.io/docs). | Key | Description | |-----|-------------| | privateRegistry.createSecret | Set to `true` to automatically create a new private registry secret. | | privateRegistry.registryPasswd | Password for authenticating with a private registry. | | privateRegistry.registrySecret | If create a new private registry secret is true, create a Kubernetes secret with this name; else use the existing secret of this name. Use it to pull images from your private registry. | | privateRegistry.registryUrl | URL of a private registry. When unspecified, Longhorn uses the default system registry. | | privateRegistry.registryUser | User account used for authenticating with a private registry. | ### Metrics Settings | Key | Type | Default | Description | |-----|------|---------|-------------| | metrics.serviceMonitor.additionalLabels | object | `{}` | Additional labels for the Prometheus ServiceMonitor resource. | | metrics.serviceMonitor.annotations | object | `{}` | Annotations for the Prometheus ServiceMonitor resource. | | metrics.serviceMonitor.enabled | bool | `false` | Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components. | | metrics.serviceMonitor.interval | string | `""` | Interval at which Prometheus scrapes the metrics from the target. | | metrics.serviceMonitor.metricRelabelings | list | `[]` | Configures the relabeling rules to apply to the samples before ingestion. See the [Prometheus Operator documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for formatting details. | | metrics.serviceMonitor.relabelings | list | `[]` | Configures the relabeling rules to apply the target’s metadata labels. See the [Prometheus Operator documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for formatting details. | | metrics.serviceMonitor.scrapeTimeout | string | `""` | Timeout after which Prometheus considers the scrape to be failed. | ### OS/Kubernetes Distro Settings #### OpenShift Settings For more details, see the [ocp-readme](https://github.com/longhorn/longhorn/blob/master/chart/ocp-readme.md). | Key | Type | Default | Description | |-----|------|---------|-------------| | openshift.enabled | bool | `false` | Setting that allows Longhorn to integrate with OpenShift. | | openshift.ui.port | int | `443` | Port for accessing the OpenShift web console. | | openshift.ui.proxy | int | `8443` | Port for proxy that provides access to the OpenShift web console. | | openshift.ui.route | string | `"longhorn-ui"` | Route for connections between Longhorn and the OpenShift web console. | ### Other Settings | Key | Default | Description | |-----|---------|-------------| | annotations | `{}` | Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. | | defaultBackupStore | `{"backupTarget":null,"backupTargetCredentialSecret":null,"pollInterval":null}` | Setting that allows you to update the default backupstore. | | defaultBackupStore.backupTarget | `nil` | Endpoint used to access the default backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") | | defaultBackupStore.backupTargetCredentialSecret | `nil` | Name of the Kubernetes secret associated with the default backup target. | | defaultBackupStore.pollInterval | `nil` | Number of seconds that Longhorn waits before checking the default backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. | | enableGoCoverDir | `false` | Setting that allows Longhorn to generate code coverage profiles. | | enablePSP | `false` | Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. | | extraObjects | `[]` | Add extra objects manifests | | namespaceOverride | `""` | Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`. | | preUpgradeChecker.jobEnabled | `true` | Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions. | | preUpgradeChecker.upgradeVersionCheck | `true` | Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled. | ### System Default Settings During installation, you can either allow Longhorn to use the default system settings or use specific flags to modify the default values. After installation, you can modify the settings using the Longhorn UI. For more information, see the **Settings Reference** section of the [documentation](https://longhorn.io/docs). | Key | Description | |-----|-------------| | defaultSettings.allowCollectingLonghornUsageMetrics | Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. | | defaultSettings.allowEmptyDiskSelectorVolume | Setting that allows scheduling of empty disk selector volumes to any disk. | | defaultSettings.allowEmptyNodeSelectorVolume | Setting that allows scheduling of empty node selector volumes to any node. | | defaultSettings.allowRecurringJobWhileVolumeDetached | Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. | | defaultSettings.allowVolumeCreationWithDegradedAvailability | Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. | | defaultSettings.autoCleanupRecurringJobBackupSnapshot | Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. | | defaultSettings.autoCleanupSnapshotAfterOnDemandBackupCompleted | Setting that automatically cleans up the snapshot after the on-demand backup is completed. | | defaultSettings.autoCleanupSnapshotWhenDeleteBackup | Setting that automatically cleans up the snapshot when the backup is deleted. | | defaultSettings.autoCleanupSystemGeneratedSnapshot | Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. | | defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly | Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. | | defaultSettings.autoSalvage | Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. | | defaultSettings.backingImageCleanupWaitInterval | Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. | | defaultSettings.backingImageRecoveryWaitInterval | Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". | | defaultSettings.backupCompressionMethod | Setting that allows you to specify a backup compression method. | | defaultSettings.backupConcurrentLimit | Maximum number of worker threads that can concurrently run for each backup. | | defaultSettings.backupExecutionTimeout | Number of minutes that Longhorn allows for the backup execution. The default value is "1". | | defaultSettings.blacklistForAutoDeletePodWhenVolumeDetachedUnexpectedly | Blacklist of controller api/kind values for the setting Automatically Delete Workload Pod when the Volume Is Detached Unexpectedly. If a workload pod is managed by a controller whose api/kind is listed in this blacklist, Longhorn will not automatically delete the pod when its volume is unexpectedly detached. Multiple controller api/kind entries can be specified, separated by semicolons. For example: `apps/StatefulSet;apps/DaemonSet`. Note that the controller api/kind is case sensitive and must exactly match the api/kind in the workload pod's owner reference. | | defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit | Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. | | defaultSettings.concurrentReplicaRebuildPerNodeLimit | Maximum number of replicas that can be concurrently rebuilt on each node. | | defaultSettings.concurrentVolumeBackupRestorePerNodeLimit | Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. | | defaultSettings.createDefaultDiskLabeledNodes | Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. | | defaultSettings.csiAllowedTopologyKeys | Comma-separated list of topology keys that the Longhorn CSI driver is allowed to pass through. When empty (default), no topology keys are passed through, and PVs will have no nodeAffinity. When configured (e.g., "topology.kubernetes.io/zone,topology.kubernetes.io/region"), only the specified keys are kept in topology segments. All other keys are filtered out from both CreateVolumeResponse.AccessibleTopology and NodeGetInfo topology. | | defaultSettings.dataEngineCPUMask | Applies only to the V2 Data Engine. Specifies the CPU cores on which the Storage Performance Development Kit (SPDK) target daemon runs. The daemon is deployed in each Instance Manager pod. Ensure that the number of assigned cores does not exceed the guaranteed Instance Manager CPUs for the V2 Data Engine. The default value is "{"v2":"0x1"}". | | defaultSettings.dataEngineHugepageEnabled | Applies only to the V2 Data Engine. Enables hugepages for the Storage Performance Development Kit (SPDK) target daemon. If disabled, legacy memory is used. Allocation size is set via the Data Engine Memory Size setting. | | defaultSettings.dataEngineLogFlags | Applies only to the V2 Data Engine. Specifies the log flags for the Storage Performance Development Kit (SPDK) target daemon. | | defaultSettings.dataEngineLogLevel | Applies only to the V2 Data Engine. Specifies the log level for the Storage Performance Development Kit (SPDK) target daemon. Supported values are: Error, Warning, Notice, Info, and Debug. The default is Notice. | | defaultSettings.dataEngineMemorySize | Applies only to the V2 Data Engine. Specifies the hugepage size, in MiB, for the Storage Performance Development Kit (SPDK) target daemon. The default value is "{"v2":"2048"}" | | defaultSettings.defaultBackupBlockSize | Specifies the default backup block size, in MiB, used when creating a new volume. Supported values are 2 or 16. | | defaultSettings.defaultDataLocality | Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. | | defaultSettings.defaultDataPath | Default path to use for storing data on a host. An absolute directory path indicates a filesystem-type disk used by the V1 Data Engine, while a path to a block device indicates a block-type disk used by the V2 Data Engine. The default value is "/var/lib/longhorn/". | | defaultSettings.defaultLonghornStaticStorageClass | Default name of Longhorn static StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. "storageClassName" needs to be an existing StorageClass. The default value is "longhorn-static". | | defaultSettings.defaultReplicaCount | Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "{"v1":"3","v2":"3"}". | | defaultSettings.defaultUblkNumberOfQueue | This setting specifies the default the number of queues for ublk frontend. This setting applies to volumes using the V2 Data Engine with Ublk front end. Individual volumes can override this setting by specifying their own number of queues for ublk. | | defaultSettings.defaultUblkQueueDepth | This setting specifies the default depth of each queue for Ublk frontend. This setting applies to volumes using the V2 Data Engine with Ublk front end. Individual volumes can override this setting by specifying their own Ublk queue depth. | | defaultSettings.deletingConfirmationFlag | Flag that prevents accidental uninstallation of Longhorn. | | defaultSettings.detachManuallyAttachedVolumesWhenCordoned | Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. | | defaultSettings.disableRevisionCounter | Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. | | defaultSettings.disableSchedulingOnCordonedNode | Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. | | defaultSettings.disableSnapshotPurge | Setting that temporarily prevents all attempts to purge volume snapshots. | | defaultSettings.endpointNetworkForRWXVolume | Specifies a dedicated network for mounting RWX (ReadWriteMany) volumes. Leave this blank to use the default Kubernetes cluster network. **Caution**: This setting should change after all RWX volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while RWX volumes are still attached. | | defaultSettings.engineReplicaTimeout | Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". | | defaultSettings.failedBackupTTL | Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. | | defaultSettings.fastReplicaRebuildEnabled | Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". | | defaultSettings.freezeFilesystemForSnapshot | Setting that freezes the filesystem on the root partition before a snapshot is created. | | defaultSettings.guaranteedInstanceManagerCPU | Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod. The default value is {"v1":"12","v2":"12"}. | | defaultSettings.instanceManagerPodLivenessProbeTimeout | In seconds. The setting specifies the timeout for the instance manager pod liveness probe. The default value is 10 seconds. | | defaultSettings.kubernetesClusterAutoscalerEnabled | Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. | | defaultSettings.logLevel | Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") | | defaultSettings.logPath | Specifies the directory on the host where Longhorn stores log files for the instance manager pod. Currently, it is only used for instance manager pods in the v2 data engine. | | defaultSettings.longGRPCTimeOut | Number of seconds that Longhorn allows for the completion of replica rebuilding and snapshot cloning operations. | | defaultSettings.managerUrl | The external URL used to access the Longhorn Manager API. When set, this URL is returned in API responses (the actions and links fields) instead of the internal pod IP. This is useful when accessing the API through Ingress or Gateway API HTTPRoute. Format: scheme://host[:port] (for example, https://longhorn.example.com or https://longhorn.example.com:8443). Leave it empty to use the default behavior. | | defaultSettings.nodeDiskHealthMonitoring | Controls whether Longhorn monitors and records health information for node disks. When disabled, disk health checks and status updates are skipped. | | defaultSettings.nodeDownPodDeletionPolicy | Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. | | defaultSettings.nodeDrainPolicy | Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. | | defaultSettings.offlineReplicaRebuilding | Enables automatic rebuilding of degraded replicas while the volume is detached. This setting only takes effect if the individual volume setting is set to `ignored` or `enabled`. | | defaultSettings.orphanResourceAutoDeletion | Enables Longhorn to automatically delete orphaned resources and their associated data or processes (e.g., stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. You need to specify the resource types to be deleted using a semicolon-separated list (e.g., `replica-data;instance`). Available items are: `replica-data`, `instance`. | | defaultSettings.orphanResourceAutoDeletionGracePeriod | Specifies the wait time, in seconds, before Longhorn automatically deletes an orphaned Custom Resource (CR) and its associated resources. Note that if a user manually deletes an orphaned CR, the deletion occurs immediately and does not respect this grace period. | | defaultSettings.priorityClass | PriorityClass for system-managed Longhorn components. This setting can help prevent Longhorn components from being evicted under Node Pressure. Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. | | defaultSettings.rebuildConcurrentSyncLimit | Maximum number of file synchronization operations that can run concurrently during a single replica rebuild. Right now, it's for v1 data engine only. | | defaultSettings.recurringFailedJobsHistoryLimit | Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. | | defaultSettings.recurringJobMaxRetention | Maximum number of snapshots or backups to be retained. | | defaultSettings.recurringSuccessfulJobsHistoryLimit | Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained. | | defaultSettings.removeSnapshotsDuringFilesystemTrim | Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files. | | defaultSettings.replicaAutoBalance | Setting that automatically rebalances replicas when an available node is discovered. | | defaultSettings.replicaDiskSoftAntiAffinity | Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default. | | defaultSettings.replicaFileSyncHttpClientTimeout | Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed. | | defaultSettings.replicaRebuildingBandwidthLimit | This setting specifies the default write bandwidth limit (in megabytes per second) for volume replica rebuilding when using the v2 data engine (SPDK). If this value is set to 0, there will be no write bandwidth limitation. Individual volumes can override this setting by specifying their own rebuilding bandwidth limit. | | defaultSettings.replicaReplenishmentWaitInterval | Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume. | | defaultSettings.replicaSoftAntiAffinity | Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default. | | defaultSettings.replicaZoneSoftAntiAffinity | Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=" in the Kubernetes node object. | | defaultSettings.restoreConcurrentLimit | Maximum number of worker threads that can concurrently run for each restore operation. | | defaultSettings.restoreVolumeRecurringJobs | Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration. | | defaultSettings.rwxVolumeFastFailover | Setting that allows Longhorn to detect node failure and immediately migrate affected RWX volumes. | | defaultSettings.snapshotDataIntegrity | Setting that allows you to enable and disable snapshot hashing and data integrity checks. | | defaultSettings.snapshotDataIntegrityCronjob | Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format. | | defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation | Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance. | | defaultSettings.snapshotHeavyTaskConcurrentLimit | Setting that controls how many snapshot heavy task operations (such as purge and clone) can run concurrently per node. This is a best-effort mechanism: due to the distributed nature of the system, temporary oversubscription may occur. The limiter reduces worst-case overload but does not guarantee perfect enforcement. | | defaultSettings.snapshotMaxCount | Maximum snapshot count for a volume. The value should be between 2 to 250 | | defaultSettings.storageMinimalAvailablePercentage | Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25". | | defaultSettings.storageNetwork | Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network. | | defaultSettings.storageOverProvisioningPercentage | Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100". | | defaultSettings.storageReservedPercentageForDefaultDisk | Percentage of disk space that is not allocated to the default disk on each new Longhorn node. | | defaultSettings.supportBundleFailedHistoryLimit | Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles. | | defaultSettings.systemManagedCSIComponentsResourceLimits | Resource limits for system-managed CSI components. This setting allows you to configure CPU and memory requests/limits for CSI attacher, provisioner, resizer, snapshotter, and plugin components. Supported components: csi-attacher, csi-provisioner, csi-resizer, csi-snapshotter, longhorn-csi-plugin, node-driver-registrar, longhorn-liveness-probe. Notice that changing resource limits will cause CSI components to restart, which may temporarily affect volume provisioning and attach/detach operations until the components are ready. The value should be a JSON object with component names as keys and ResourceRequirements as values. | | defaultSettings.systemManagedComponentsNodeSelector | Node selector for system-managed Longhorn components. | | defaultSettings.systemManagedPodsImagePullPolicy | Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. | | defaultSettings.taintToleration | Taint or toleration for system-managed Longhorn components. Specify values using a semicolon-separated list in `kubectl taint` syntax (Example: key1=value1:effect; key2=value2:effect). | | defaultSettings.upgradeChecker | Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default | | defaultSettings.upgradeResponderURL | The Upgrade Responder sends a notification whenever a new Longhorn version that you can upgrade to becomes available. The default value is https://longhorn-upgrade-responder.rancher.io/v1/checkupgrade. | | defaultSettings.v1DataEngine | Setting that allows you to enable the V1 Data Engine. | | defaultSettings.v2DataEngine | Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is an experimental feature and should not be used in production environments. | --- Please see [link](https://github.com/longhorn/longhorn) for more information. ================================================ FILE: chart/README.md.gotmpl ================================================ # Longhorn Chart > **Important**: Please install the Longhorn chart in the `longhorn-system` namespace only. > **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. > **Note**: Use Helm 3 when installing and upgrading Longhorn. Helm 2 is [no longer supported](https://helm.sh/blog/helm-2-becomes-unsupported/). ## Source Code Longhorn is 100% open source software. Project source code is spread across a number of repos: 1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine 2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager 3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager 4. Backing Image Manager -- Backing image file lifecycle management. https://github.com/longhorn/backing-image-manager 5. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager 6. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui ## Prerequisites 1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) 2. Kubernetes >= v1.25 3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. 4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. ## Upgrading to Kubernetes v1.25+ Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. > **Note:** > If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** > > If your charts get stuck in this state, you may have to clean up your Helm release secrets. Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Longhorn docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. ## Installation 1. Add Longhorn chart repository. ``` helm repo add longhorn https://charts.longhorn.io ``` 2. Update local Longhorn chart information from chart repository. ``` helm repo update ``` 3. Use the following commands to create the `longhorn-system` namespace first, then install the Longhorn chart. ``` kubectl create namespace longhorn-system helm install longhorn longhorn/longhorn --namespace longhorn-system ``` ## Uninstallation ``` kubectl -n longhorn-system patch -p '{"value": "true"}' --type=merge lhs deleting-confirmation-flag helm uninstall longhorn -n longhorn-system kubectl delete namespace longhorn-system ``` ## Values The `values.yaml` contains items used to tweak a deployment of this chart. ### Cattle Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "global" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Network Policies | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "networkPolicies" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Image Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "image" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Service Settings | Key | Description | |-----|-------------| {{- range .Values }} {{- if (and (hasPrefix "service" .Key) (not (contains "Account" .Key))) }} | {{ .Key }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### StorageClass Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "persistence" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### CSI Settings | Key | Description | |-----|-------------| {{- range .Values }} {{- if hasPrefix "csi" .Key }} | {{ .Key }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Longhorn Manager Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn Manager. | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "longhornManager" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Longhorn Driver Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn Driver. | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "longhornDriver" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Longhorn UI Settings Longhorn consists of user-deployed components (for example, Longhorn Manager, Longhorn Driver, and Longhorn UI) and system-managed components (for example, Instance Manager, Backing Image Manager, Share Manager, CSI Driver, and Engine Image). The following settings only apply to Longhorn UI. | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "longhornUI" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Ingress Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "ingress" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### HTTPRoute Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "httproute" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Private Registry Settings You can install Longhorn in an air-gapped environment with a private registry. For more information, see the **Air Gap Installation** section of the [documentation](https://longhorn.io/docs). | Key | Description | |-----|-------------| {{- range .Values }} {{- if hasPrefix "privateRegistry" .Key }} | {{ .Key }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Metrics Settings | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "metrics" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### OS/Kubernetes Distro Settings #### OpenShift Settings For more details, see the [ocp-readme](https://github.com/longhorn/longhorn/blob/master/chart/ocp-readme.md). | Key | Type | Default | Description | |-----|------|---------|-------------| {{- range .Values }} {{- if hasPrefix "openshift" .Key }} | {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### Other Settings | Key | Default | Description | |-----|---------|-------------| {{- range .Values }} {{- if not (or (hasPrefix "defaultSettings" .Key) (hasPrefix "networkPolicies" .Key) (hasPrefix "image" .Key) (hasPrefix "service" .Key) (hasPrefix "persistence" .Key) (hasPrefix "csi" .Key) (hasPrefix "longhornManager" .Key) (hasPrefix "longhornDriver" .Key) (hasPrefix "longhornUI" .Key) (hasPrefix "privateRegistry" .Key) (hasPrefix "ingress" .Key) (hasPrefix "httproute" .Key) (hasPrefix "metrics" .Key) (hasPrefix "openshift" .Key) (hasPrefix "global" .Key)) }} | {{ .Key }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} ### System Default Settings During installation, you can either allow Longhorn to use the default system settings or use specific flags to modify the default values. After installation, you can modify the settings using the Longhorn UI. For more information, see the **Settings Reference** section of the [documentation](https://longhorn.io/docs). | Key | Description | |-----|-------------| {{- range .Values }} {{- if hasPrefix "defaultSettings" .Key }} | {{ .Key }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} | {{- end }} {{- end }} --- Please see [link](https://github.com/longhorn/longhorn) for more information. ================================================ FILE: chart/app-readme.md ================================================ # Longhorn Longhorn is a lightweight, reliable and easy to use distributed block storage system for Kubernetes. Once deployed, users can leverage persistent volumes provided by Longhorn. Longhorn creates a dedicated storage controller for each volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. The storage controller and replicas are themselves orchestrated using Kubernetes. Longhorn supports snapshots, backups and even allows you to schedule recurring snapshots and backups! **Important**: Please install Longhorn chart in `longhorn-system` namespace only. **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. [Chart Documentation](https://github.com/longhorn/longhorn/blob/master/chart/README.md) ================================================ FILE: chart/ocp-readme.md ================================================ # OpenShift / OKD Extra Configuration Steps - [OpenShift / OKD Extra Configuration Steps](#openshift--okd-extra-configuration-steps) - [Notes](#notes) - [Known Issues](#known-issues) - [Preparing Nodes (Optional)](#preparing-nodes-optional) - [Default /var/lib/longhorn setup](#default-varliblonghorn-setup) - [Separate /var/mnt/longhorn setup](#separate-varmntlonghorn-setup) - [Create Filesystem](#create-filesystem) - [Mounting Disk On Boot](#mounting-disk-on-boot) - [Label and Annotate Nodes](#label-and-annotate-nodes) - [Example values.yaml](#example-valuesyaml) - [Installation](#installation) - [Refs](#refs) ## Notes Main changes and tasks for OCP are: - On OCP / OKD, the Operating System is Managed by the Cluster - OCP Imposes [Security Context Constraints](https://docs.openshift.com/container-platform/4.11/authentication/managing-security-context-constraints.html) - This requires everything to run with the least privilege possible. For the moment every component has been given access to run as higher privilege. - Something to circle back on is network polices and which components can have their privileges reduced without impacting functionality. - The UI probably can be for example. - openshift/oauth-proxy for authentication to the Longhorn Ui - **⚠️** Currently Scoped to Authenticated Users that can delete a longhorn settings object. - **⚠️** Since the UI it self is not protected, network policies will need to be created to prevent namespace <--> namespace communication against the pod or service object directly. - Anyone with access to the UI Deployment can remove the route restriction. (Namespace Scoped Admin) - Option to use separate disk in /var/mnt/longhorn & MachineConfig file to mount /var/mnt/longhorn - Adding finalizers for mount propagation ## Known Issues - General Feature/Issue Thread - [[FEATURE] Deploying Longhorn on OKD/Openshift](https://github.com/longhorn/longhorn/issues/1831) - 4.10 / 1.23: - 4.10.0-0.okd-2022-03-07-131213 to 4.10.0-0.okd-2022-07-09-073606 - Tested, No Known Issues - 4.11 / 1.24: - 4.11.0-0.okd-2022-07-27-052000 to 4.11.0-0.okd-2022-11-19-050030 - Tested, No Known Issues - 4.11.0-0.okd-2022-12-02-145640, 4.11.0-0.okd-2023-01-14-152430: - Workaround: [[BUG] Volumes Stuck in Attach/Detach Loop](https://github.com/longhorn/longhorn/issues/4988) - [MachineConfig Patch](https://github.com/longhorn/longhorn/issues/4988#issuecomment-1345676772) - 4.12 / 1.25: - 4.12.0-0.okd-2022-12-05-210624 to 4.12.0-0.okd-2023-01-20-101927 - Tested, No Known Issues - 4.12.0-0.okd-2023-01-21-055900 to 4.12.0-0.okd-2023-02-18-033438: - Workaround: [[BUG] Volumes Stuck in Attach/Detach Loop](https://github.com/longhorn/longhorn/issues/4988) - [MachineConfig Patch](https://github.com/longhorn/longhorn/issues/4988#issuecomment-1345676772) - 4.12.0-0.okd-2023-03-05-022504 - 4.12.0-0.okd-2023-04-16-041331: - Tested, No Known Issues - 4.13 / 1.26: - 4.13.0-0.okd-2023-05-03-001308 - 4.13.0-0.okd-2023-08-18-135805: - Tested, No Known Issues - 4.14 / 1.27: - 4.14.0-0.okd-2023-08-12-022330 - 4.14.0-0.okd-2023-10-28-073550: - Tested, No Known Issues ## Preparing Nodes (Optional) Only required if you require additional customizations, such as storage-less nodes, or secondary disks. ### Default /var/lib/longhorn setup Label each node for storage with: ```bash oc get nodes --no-headers | awk '{print $1}' export NODE="worker-0" oc label node "${NODE}" node.longhorn.io/create-default-disk=true ``` ### Separate /var/mnt/longhorn setup #### Create Filesystem On the storage nodes create a filesystem with the label longhorn: ```bash oc get nodes --no-headers | awk '{print $1}' export NODE="worker-0" oc debug node/${NODE} -t -- chroot /host bash # Validate Target Drive is Present lsblk export DRIVE="sdb" #vdb sudo mkfs.ext4 -L longhorn /dev/${DRIVE} ``` > ⚠️ Note: If you add New Nodes After the below Machine Config is applied, you will need to also reboot the node. #### Mounting Disk On Boot The Secondary Drive needs to be mounted on every boot. Save the Concents and Apply the MachineConfig with `oc apply -f`: > ⚠️ This will trigger an machine config profile update and reboot all worker nodes on the cluster ```yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: 71-mount-storage-worker spec: config: ignition: version: 3.2.0 systemd: units: - name: var-mnt-longhorn.mount enabled: true contents: | [Unit] Before=local-fs.target [Mount] Where=/var/mnt/longhorn What=/dev/disk/by-label/longhorn Options=rw,relatime,discard [Install] WantedBy=local-fs.target ``` #### Label and Annotate Nodes Label and annotate storage nodes like this: ```bash oc get nodes --no-headers | awk '{print $1}' export NODE="worker-0" oc annotate node ${NODE} --overwrite node.longhorn.io/default-disks-config='[{"path":"/var/mnt/longhorn","allowScheduling":true}]' oc label node ${NODE} node.longhorn.io/create-default-disk=config ``` ## Example values.yaml Minimum Adjustments Required ```yaml image: openshift: oauthProxy: repository: quay.io/openshift/origin-oauth-proxy tag: 4.18 # Use Your OCP/OKD 4.X Version, Current Stable is 4.18 # defaultSettings: # Preparing nodes (Optional) # createDefaultDiskLabeledNodes: true openshift: enabled: true ui: route: "longhorn-ui" port: 443 proxy: 8443 ``` ## Installation ```bash # helm template ./chart/ --namespace longhorn-system --values ./chart/values.yaml --no-hooks > longhorn.yaml # Local Testing helm template longhorn --namespace longhorn-system --values values.yaml --no-hooks > longhorn.yaml oc create namespace longhorn-system -o yaml --dry-run=client | oc apply -f - oc apply -f longhorn.yaml -n longhorn-system ``` ## Refs - - - okd 4.5: - okd 4.6: - oauth-proxy: - ================================================ FILE: chart/questions.yaml ================================================ categories: - storage namespace: longhorn-system questions: - variable: image.defaultImage default: 'true' description: Use default Longhorn images label: Use Default Images type: boolean show_subquestion_if: false group: Longhorn Images subquestions: - variable: image.longhorn.manager.repository default: longhornio/longhorn-manager description: Repository for the Longhorn Manager image. type: string label: Longhorn Manager Image Repository group: Longhorn Images Settings - variable: image.longhorn.manager.tag default: master-head description: Tag for the Longhorn Manager image. type: string label: Longhorn Manager Image Tag group: Longhorn Images Settings - variable: image.longhorn.engine.repository default: longhornio/longhorn-engine description: Repository for the Longhorn Engine image. type: string label: Longhorn Engine Image Repository group: Longhorn Images Settings - variable: image.longhorn.engine.tag default: master-head description: Tag for the Longhorn Engine image. type: string label: Longhorn Engine Image Tag group: Longhorn Images Settings - variable: image.longhorn.ui.repository default: longhornio/longhorn-ui description: Repository for the Longhorn UI image. type: string label: Longhorn UI Image Repository group: Longhorn Images Settings - variable: image.longhorn.ui.tag default: master-head description: Tag for the Longhorn UI image. type: string label: Longhorn UI Image Tag group: Longhorn Images Settings - variable: image.longhorn.instanceManager.repository default: longhornio/longhorn-instance-manager description: Repository for the Longhorn Instance Manager image. type: string label: Longhorn Instance Manager Image Repository group: Longhorn Images Settings - variable: image.longhorn.instanceManager.tag default: master-head description: Tag for the Longhorn Instance Manager image. type: string label: Longhorn Instance Manager Image Tag group: Longhorn Images Settings - variable: image.longhorn.shareManager.repository default: longhornio/longhorn-share-manager description: Repository for the Longhorn Share Manager image. type: string label: Longhorn Share Manager Image Repository group: Longhorn Images Settings - variable: image.longhorn.shareManager.tag default: master-head description: Tag for the Longhorn Share Manager image. type: string label: Longhorn Share Manager Image Tag group: Longhorn Images Settings - variable: image.longhorn.backingImageManager.repository default: longhornio/backing-image-manager description: >- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. type: string label: Longhorn Backing Image Manager Image Repository group: Longhorn Images Settings - variable: image.longhorn.backingImageManager.tag default: master-head description: >- Tag for the Backing Image Manager image. When unspecified, Longhorn uses the default value. type: string label: Longhorn Backing Image Manager Image Tag group: Longhorn Images Settings - variable: image.longhorn.supportBundleKit.repository default: longhornio/support-bundle-kit description: Repository for the Longhorn Support Bundle Manager image. type: string label: Longhorn Support Bundle Kit Image Repository group: Longhorn Images Settings - variable: image.longhorn.supportBundleKit.tag default: v0.0.79 description: Tag for the Longhorn Support Bundle Manager image. type: string label: Longhorn Support Bundle Kit Image Tag group: Longhorn Images Settings - variable: image.csi.attacher.repository default: longhornio/csi-attacher description: >- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Attacher Image Repository group: Longhorn CSI Driver Images - variable: image.csi.attacher.tag default: v4.10.0-20251226 description: >- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Attacher Image Tag group: Longhorn CSI Driver Images - variable: image.csi.provisioner.repository default: longhornio/csi-provisioner description: >- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Provisioner Image Repository group: Longhorn CSI Driver Images - variable: image.csi.provisioner.tag default: v5.3.0-20251226 description: >- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Provisioner Image Tag group: Longhorn CSI Driver Images - variable: image.csi.nodeDriverRegistrar.repository default: longhornio/csi-node-driver-registrar description: >- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Node Driver Registrar Image Repository group: Longhorn CSI Driver Images - variable: image.csi.nodeDriverRegistrar.tag default: v2.15.0-20251226 description: >- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Node Driver Registrar Image Tag group: Longhorn CSI Driver Images - variable: image.csi.resizer.repository default: longhornio/csi-resizer description: >- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Driver Resizer Image Repository group: Longhorn CSI Driver Images - variable: image.csi.resizer.tag default: v2.0.0-20251226 description: >- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Driver Resizer Image Tag group: Longhorn CSI Driver Images - variable: image.csi.snapshotter.repository default: longhornio/csi-snapshotter description: >- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Driver Snapshotter Image Repository group: Longhorn CSI Driver Images - variable: image.csi.snapshotter.tag default: v8.4.0-20251226 description: >- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Driver Snapshotter Image Tag group: Longhorn CSI Driver Images - variable: image.csi.livenessProbe.repository default: longhornio/livenessprobe description: >- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Liveness Probe Image Repository group: Longhorn CSI Driver Images - variable: image.csi.livenessProbe.tag default: v2.17.0-20251226 description: >- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. type: string label: Longhorn CSI Liveness Probe Image Tag group: Longhorn CSI Driver Images - variable: privateRegistry.registryUrl label: Private registry URL description: >- URL of a private registry. When unspecified, Longhorn uses the default system registry. group: Private Registry Settings type: string default: '' subquestions: [] - variable: privateRegistry.registrySecret label: Private registry secret name description: >- If create a new private registry secret is true, create a Kubernetes secret with this name; else use the existing secret of this name. Use it to pull images from your private registry. group: Private Registry Settings type: string default: '' subquestions: [] - variable: privateRegistry.createSecret default: 'true' description: Set to true to automatically create a new private registry secret. type: boolean group: Private Registry Settings label: Create Secret for Private Registry Settings show_subquestion_if: true subquestions: - variable: privateRegistry.registryUser label: Private registry user description: User account used for authenticating with a private registry. type: string default: '' - variable: privateRegistry.registryPasswd label: Private registry password description: Password for authenticating with a private registry. type: password default: '' - variable: longhorn.default_setting default: 'false' description: >- Customize the default settings before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn. label: Customize Default Settings type: boolean show_subquestion_if: true group: Longhorn Default Settings subquestions: - variable: csi.kubeletRootDir default: null description: >- kubelet root directory. When unspecified, Longhorn uses the default value. type: string label: Kubelet Root Directory group: Longhorn CSI Driver Settings - variable: csi.podAntiAffinityPreset type: string label: Longhorn CSI Pod AntiAffinity Preset group: Longhorn CSI Driver Settings default: soft description: >- Configures Pod anti-affinity to prevent multiple instances on the same node. Use soft (tries to separate) or hard (must separate). When unspecified, Longhorn uses the default value ("soft"). - variable: csi.attacherReplicaCount type: int default: 3 min: 1 max: 10 description: >- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). label: Longhorn CSI Attacher replica count group: Longhorn CSI Driver Settings - variable: csi.provisionerReplicaCount type: int default: 3 min: 1 max: 10 description: >- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). label: Longhorn CSI Provisioner replica count group: Longhorn CSI Driver Settings - variable: csi.resizerReplicaCount type: int default: 3 min: 1 max: 10 description: >- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). label: Longhorn CSI Resizer replica count group: Longhorn CSI Driver Settings - variable: csi.snapshotterReplicaCount type: int default: 3 min: 1 max: 10 description: >- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). label: Longhorn CSI Snapshotter replica count group: Longhorn CSI Driver Settings - variable: defaultSettings.allowRecurringJobWhileVolumeDetached label: Allow Recurring Job While Volume Is Detached description: >- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.snapshotMaxCount label: Snapshot Maximum Count description: >- Maximum snapshot count for a volume. The value should be between 2 to 250. group: Longhorn Default Settings type: int min: 2 max: 250 default: 250 - variable: defaultSettings.createDefaultDiskLabeledNodes label: Create Default Disk on Labeled Nodes description: >- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.defaultDataPath label: Default Data Path description: >- Default path to use for storing data on a host. An absolute directory path indicates a filesystem-type disk used by the V1 Data Engine, while a path to a block device indicates a block-type disk used by the V2 Data Engine. The default value is "/var/lib/longhorn/". group: Longhorn Default Settings type: string default: /var/lib/longhorn/ - variable: defaultSettings.defaultDataLocality label: Default Data Locality description: >- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. group: Longhorn Default Settings type: enum options: - disabled - best-effort default: disabled - variable: defaultSettings.replicaSoftAntiAffinity label: Replica Node Level Soft Anti-Affinity description: >- Allow scheduling on nodes with existing healthy replicas of the same volume. By default, false. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.replicaAutoBalance label: Replica Auto Balance description: >- Enable this setting automatically re-balances replicas when discovered an available node. group: Longhorn Default Settings type: enum options: - disabled - least-effort - best-effort default: disabled - variable: defaultSettings.storageOverProvisioningPercentage label: Storage Over Provisioning Percentage description: >- Percentage of storage that can be allocated relative to hard drive capacity. The default value is 100. group: Longhorn Default Settings type: int min: 0 default: 100 - variable: defaultSettings.storageMinimalAvailablePercentage label: Storage Minimal Available Percentage description: >- If the minimum available disk capacity exceeds the actual percentage of available disk capacity, the disk becomes unschedulable until more space is freed up. By default, 25. group: Longhorn Default Settings type: int min: 0 max: 100 default: 25 - variable: defaultSettings.storageReservedPercentageForDefaultDisk label: Storage Reserved Percentage For Default Disk description: >- The reserved percentage specifies the percentage of disk space that will not be allocated to the default disk on each new Longhorn node. group: Longhorn Default Settings type: int min: 0 max: 100 default: 30 - variable: defaultSettings.upgradeChecker label: Enable Upgrade Checker description: >- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.upgradeResponderURL label: Upgrade Responder URL description: >- The Upgrade Responder sends a notification whenever a new Longhorn version that you can upgrade to becomes available. The default value is "https://longhorn-upgrade-responder.rancher.io/v1/checkupgrade". group: Longhorn Default Settings type: string default: 'https://longhorn-upgrade-responder.rancher.io/v1/checkupgrade' - variable: defaultSettings.managerUrl label: Manager URL description: >- The external URL used to access the Longhorn Manager API. When set, this URL is returned in API responses (the actions and links fields) instead of the internal pod IP. This is useful when accessing the API through Ingress or Gateway API HTTPRoute. Format: scheme://host[:port] (for example, https://longhorn.example.com or https://longhorn.example.com:8443). Leave it empty to use the default behavior. group: Longhorn Default Settings type: string default: '' - variable: defaultSettings.defaultReplicaCount label: Default Replica Count description: >- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. group: Longhorn Default Settings type: string default: '{"v1":"3","v2":"3"}' - variable: defaultSettings.defaultLonghornStaticStorageClass label: Default Longhorn Static StorageClass Name description: >- Default name of Longhorn Static StorageClass. The "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. The "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. The "storageClassName" needs to be an existing StorageClass. The default value is "longhorn-static". group: Longhorn Default Settings type: string default: longhorn-static - variable: defaultSettings.failedBackupTTL label: Failed Backup Time to Live description: >- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. group: Longhorn Default Settings type: int min: 0 default: 1440 - variable: defaultSettings.backupExecutionTimeout label: Backup Execution Timeout description: >- Number of minutes that Longhorn allows for the backup execution. The default value is "1". group: Longhorn Default Settings type: int min: 1 default: 1 - variable: defaultSettings.restoreVolumeRecurringJobs label: Restore Volume Recurring Jobs description: >- Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.recurringSuccessfulJobsHistoryLimit label: Cronjob Successful Jobs History Limit description: >- This setting specifies how many successful backup or snapshot job histories should be retained. History will not be retained if the value is 0. group: Longhorn Default Settings type: int min: 0 default: 1 - variable: defaultSettings.recurringFailedJobsHistoryLimit label: Cronjob Failed Jobs History Limit description: >- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. group: Longhorn Default Settings type: int min: 0 default: 1 - variable: defaultSettings.recurringJobMaxRetention label: Maximum Retention Number for Recurring Job description: Maximum number of snapshots or backups to be retained. group: Longhorn Default Settings type: int default: 100 - variable: defaultSettings.supportBundleFailedHistoryLimit label: SupportBundle Failed History Limit description: >- This setting specifies how many failed support bundles can exist in the cluster. Set this value to **0** to have Longhorn automatically purge all failed support bundles. group: Longhorn Default Settings type: int min: 0 default: 1 - variable: defaultSettings.autoSalvage label: Automatic salvage description: >- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly label: >- Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly description: >- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.blacklistForAutoDeletePodWhenVolumeDetachedUnexpectedly label: >- Blacklist for Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly description: >- Blacklist of controller api/kind values for the setting Automatically Delete Workload Pod when the Volume Is Detached Unexpectedly. If a workload pod is managed by a controller whose api/kind is listed in this blacklist, Longhorn will not automatically delete the pod when its volume is unexpectedly detached. Multiple controller api/kind entries can be specified, separated by semicolons. For example: `apps/StatefulSet;apps/DaemonSet`. Note that the controller api/kind is case sensitive and must exactly match the api/kind in the workload pod's owner reference. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.disableSchedulingOnCordonedNode label: Disable Scheduling On Cordoned Node description: >- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.replicaZoneSoftAntiAffinity label: Replica Zone Level Soft Anti-Affinity description: >- Allow scheduling new Replicas of Volume to the Nodes in the same Zone as existing healthy Replicas. Nodes don't belong to any Zone will be treated as in the same Zone. Notice that Longhorn relies on label `topology.kubernetes.io/zone=` in the Kubernetes node object to identify the zone. By, default true. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.replicaDiskSoftAntiAffinity label: Replica Disk Level Soft Anti-Affinity description: >- Allow scheduling on disks with existing healthy replicas of the same volume. By default, true. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.allowEmptyNodeSelectorVolume label: Allow Empty Node Selector Volume description: >- Setting that allows scheduling of empty node selector volumes to any node. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.allowEmptyDiskSelectorVolume label: Allow Empty Disk Selector Volume description: >- Setting that allows scheduling of empty disk selector volumes to any disk. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.nodeDownPodDeletionPolicy label: Pod Deletion Policy When Node is Down description: >- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. group: Longhorn Default Settings type: enum options: - do-nothing - delete-statefulset-pod - delete-deployment-pod - delete-both-statefulset-and-deployment-pod default: do-nothing - variable: defaultSettings.nodeDrainPolicy label: Node Drain Policy description: >- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. group: Longhorn Default Settings type: enum options: - block-for-eviction - block-for-eviction-if-contains-last-replica - block-if-contains-last-replica - allow-if-replica-is-stopped - always-allow default: block-if-contains-last-replica - variable: defaultSettings.detachManuallyAttachedVolumesWhenCordoned label: Detach Manually Attached Volumes When Cordoned description: >- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.priorityClass label: Priority Class description: >- PriorityClass for system-managed Longhorn components. This setting can help prevent Longhorn components from being evicted under Node Pressure. Longhorn system contains user deployed components (E.g, Longhorn manager, Longhorn driver, Longhorn UI) and system managed components (E.g, instance manager, engine image, CSI driver, etc.) Note that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES. group: Longhorn Default Settings type: string default: longhorn-critical - variable: defaultSettings.replicaReplenishmentWaitInterval label: Replica Replenishment Wait Interval description: >- The interval in seconds determines how long Longhorn will at least wait to reuse the existing data on a failed replica rather than directly creating a new replica for a degraded volume. group: Longhorn Default Settings type: int min: 0 default: 600 - variable: defaultSettings.concurrentReplicaRebuildPerNodeLimit label: Concurrent Replica Rebuild Per Node Limit description: >- Maximum number of replicas that can be concurrently rebuilt on each node. **Caution**: [1] This setting replaces "Disable Replica Rebuild". Instead of delaying replica startup, Longhorn skips replica object replenishment to limit the number of concurrently rebuilding replicas. [2] When the value is "0", the eviction and data locality features do not work, but ongoing replica rebuilding and backup/restoration operations should remain unaffected. group: Longhorn Default Settings type: int min: 0 default: 5 - variable: defaultSettings.rebuildConcurrentSyncLimit label: Rebuild Concurrent Sync Limit description: >- Maximum number of file synchronization operations that can run concurrently during a single replica rebuild. Right now, it's for v1 data engine only. group: Longhorn Default Settings type: int min: 1 max: 5 default: 1 - variable: defaultSettings.concurrentVolumeBackupRestorePerNodeLimit label: Concurrent Volume Backup Restore Per Node Limit description: >- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. group: Longhorn Default Settings type: int min: 0 default: 5 - variable: defaultSettings.disableRevisionCounter label: Disable Revision Counter description: >- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. group: Longhorn Default Settings type: string default: '{"v1":"true"}' - variable: defaultSettings.systemManagedPodsImagePullPolicy label: System Managed Pod Image Pull Policy description: >- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. group: Longhorn Default Settings type: enum options: - if-not-present - always - never default: if-not-present - variable: defaultSettings.allowVolumeCreationWithDegradedAvailability label: Allow Volume Creation with Degraded Availability description: >- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.autoCleanupSystemGeneratedSnapshot label: Automatically Cleanup System Generated Snapshot description: >- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.autoCleanupRecurringJobBackupSnapshot label: Automatically Cleanup Recurring Job Backup Snapshot description: >- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. group: Longhorn Default Settings type: boolean default: 'true' - variable: defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit label: Concurrent Automatic Engine Upgrade Per Node Limit description: >- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. group: Longhorn Default Settings type: int min: 0 default: 0 - variable: defaultSettings.backingImageCleanupWaitInterval label: Backing Image Cleanup Wait Interval description: >- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. group: Longhorn Default Settings type: int min: 0 default: 60 - variable: defaultSettings.backingImageRecoveryWaitInterval label: Backing Image Recovery Wait Interval description: >- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". group: Longhorn Default Settings type: int min: 0 default: 300 - variable: defaultSettings.guaranteedInstanceManagerCPU label: Guaranteed Instance Manager CPU description: >- Percentage of the total allocatable CPU resources on each node to be reserved for each Instance Manager pod when the V1 Data Engine is enabled. You can specify a floating point value between "0" and "40" to accommodate the possible number of new Instance Manager pods during upgrades. The default value is "12". **Caution**: [1] When the value is "0", CPU requests are removed from the spec of Instance Manager pods. [2] A new set of Instance Manager pods must be deployed when Longhorn is upgraded. If the available CPUs are not sufficient for the new pods, you must detach the volumes using the oldest Instance Manager pods so that Longhorn can automatically clean up the old pods and release the CPU resources. Once completed, the new pods with the latest Instance Manager image are launched. [3] This global setting is not applied to nodes with a specified value for the "InstanceManagerCPURequest" field. [4] After this setting is configured, Instance Manager pods on all nodes that use this setting are automatically restarted. **Do not change this setting while volumes are still attached. [5] or the v2 Data Engine, the spdk_tgt process inside each instance manager pod uses one or more dedicated CPU cores. Setting a minimum CPU usage is critical to maintaining stability during periods of high node load.** group: Longhorn Default Settings type: string default: '{"v1":"12","v2":"12"}' - variable: defaultSettings.logLevel label: Log Level description: >- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") group: Longhorn Default Settings type: string default: Info - variable: defaultSettings.disableSnapshotPurge label: Disable Snapshot Purge description: >- Setting that temporarily prevents all attempts to purge volume snapshots. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.freezeFilesystemForSnapshot description: >- Setting that freezes the filesystem on the root partition before a snapshot is created. group: Longhorn Default Settings type: string default: '{"v1":"false"}' - variable: defaultSettings.kubernetesClusterAutoscalerEnabled label: Kubernetes Cluster Autoscaler Enabled (Experimental) description: >- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. **Caution**: Replica rebuilding may consume significant resources if the Kubernetes Cluster Autoscaler removes nodes with reusable replicas. group: Longhorn Default Settings type: boolean default: false - variable: defaultSettings.orphanResourceAutoDeletion label: Orphaned Data Cleanup description: >- Enables Longhorn to automatically delete orphaned resources and their associated data or processes (e.g., stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. You need to specify the resource types to be deleted using a semicolon-separated list (e.g., `replica-data;instance`). Valid resource types: `replica-data`, `instance`. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.orphanResourceAutoDeletionGracePeriod label: Orphaned Data Cleanup description: >- Specifies the wait time, in seconds, before Longhorn automatically deletes an orphaned Custom Resource (CR) and its associated resources. Note that if a user manually deletes an orphaned CR, the deletion occurs immediately and does not respect this grace period. group: Longhorn Default Settings type: int default: '300' - variable: defaultSettings.storageNetwork label: Storage Network description: >- Longhorn uses the storage network for in-cluster data traffic. Leave this blank to use the Kubernetes cluster network. **Caution**: This setting should change after all volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while volumes are still attached. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.EndpointNetworkForRWXVolume label: Endpoint Network For RWX Volume description: >- Specifies a dedicated network for mounting RWX (ReadWriteMany) volumes. Leave this blank to use the default Kubernetes cluster network. **Caution**: This setting should change after all RWX volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while RWX volumes are still attached. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.taintToleration label: Taint Toleration description: >- Taint or toleration for system-managed Longhorn components. Specify values using a semicolon-separated list in `kubectl taint` syntax (Example: key1=value1:effect; key2=value2:effect). **Caution**: This setting should change after all volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while volumes are still attached. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.systemManagedComponentsNodeSelector label: System Managed Components NodeSelector description: >- Node selector for system-managed Longhorn components.. **Caution**: This setting should change after all volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while volumes are still attached. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.systemManagedCSIComponentsResourceLimits label: System Managed CSI Components Resource Limits description: >- Resource limits for system managed CSI components. This setting allows you to configure CPU and memory requests/limits for CSI attacher, provisioner, resizer, snapshotter, and plugin components. Supported components: csi-attacher, csi-provisioner, csi-resizer, csi-snapshotter, longhorn-csi-plugin, node-driver-registrar, longhorn-liveness-probe. Notice that changing resource limits will cause CSI components to restart, which may temporarily affect volume provisioning and attach/detach operations until the components are ready. The value should be a JSON object with component names as keys and ResourceRequirements as values. group: Longhorn Default Settings type: string default: null - variable: defaultSettings.deletingConfirmationFlag label: Deleting Confirmation Flag description: Flag that prevents accidental uninstallation of Longhorn. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.engineReplicaTimeout label: Timeout between Engine and Replica description: >- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". group: Longhorn Default Settings type: string default: '{"v1":"8","v2":"8"}' - variable: defaultSettings.snapshotDataIntegrity label: Snapshot Data Integrity description: >- This setting allows users to enable or disable snapshot hashing and data integrity checking. group: Longhorn Default Settings type: enum options: - enabled - fast-check - disabled default: disabled - variable: >- defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation label: Immediate Snapshot Data Integrity Check After Creating a Snapshot description: >- Hashing snapshot disk files impacts the performance of the system. The immediate snapshot hashing and checking can be disabled to minimize the impact after creating a snapshot. group: Longhorn Default Settings type: string default: '{"v1":"false","v2":"false"}' - variable: defaultSettings.snapshotDataIntegrityCronjob label: Snapshot Data Integrity Check CronJob description: >- Unix-cron string format. The setting specifies when Longhorn checks the data integrity of snapshot disk files. group: Longhorn Default Settings type: string default: '{"v1":"0 0 */7 * *","v2":"0 0 */7 * *"}' - variable: defaultSettings.removeSnapshotsDuringFilesystemTrim label: Remove Snapshots During Filesystem Trim description: >- This setting allows Longhorn filesystem trim feature to automatically mark the latest snapshot and its ancestors as removed and stops at the snapshot containing multiple children. group: Longhorn Default Settings type: boolean default: 'false' - variable: defaultSettings.fastReplicaRebuildEnabled label: Fast Replica Rebuild Enabled description: >- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". group: Longhorn Default Settings type: string default: '{"v1":"true","v2":"true"}' - variable: defaultSettings.replicaFileSyncHttpClientTimeout label: Timeout of HTTP Client to Replica File Sync Server description: >- In seconds. The setting specifies the HTTP client timeout to the file sync server. group: Longhorn Default Settings type: int default: '30' - variable: defaultSettings.longGRPCTimeOut label: Long gRPC Timeout description: >- Number of seconds that Longhorn allows for the completion of replica rebuilding and snapshot cloning operations. group: Longhorn Default Settings type: int default: '86400' - variable: defaultSettings.backupCompressionMethod label: Backup Compression Method description: Setting that allows you to specify a backup compression method. group: Longhorn Default Settings type: string default: lz4 - variable: defaultSettings.backupConcurrentLimit label: Backup Concurrent Limit Per Backup description: >- Maximum number of worker threads that can concurrently run for each backup. group: Longhorn Default Settings type: int min: 1 default: 2 - variable: defaultSettings.backupBlockSize label: Backup Block Size description: >- Specifies the default backup block size, in MiB, used when creating a new volume. Supported values are 2 or 16. group: Longhorn Default Settings type: enum options: - '2' - '16' default: '2' - variable: defaultSettings.restoreConcurrentLimit label: Restore Concurrent Limit Per Backup description: >- This setting controls how many worker threads per restore concurrently. group: Longhorn Default Settings type: int min: 1 default: 2 - variable: defaultSettings.allowCollectingLonghornUsageMetrics label: Allow Collecting Longhorn Usage Metrics description: >- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. group: Longhorn Default Settings type: boolean default: true - variable: defaultSettings.v1DataEngine label: V1 Data Engine description: Setting that allows you to enable the V1 Data Engine. group: Longhorn V1 Data Engine Settings type: boolean default: true - variable: defaultSettings.v2DataEngine label: V2 Data Engine description: >- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is an experimental feature and should not be used in production environments. **Caution**: [1] **Do not modify this setting while volumes are still attached.** [2] When the V2 Data Engine is enabled, each Instance Manager pod for the V2 Data Engine uses 1 CPU core. The high CPU usage is caused by `spdk_tgt`, a process running in each Instance Manager pod that handles input/output (IO) operations and requires intensive polling. `spdk_tgt` consumes 100% of a dedicated CPU core to efficiently manage and process the IO requests, ensuring optimal performance and responsiveness for storage operations. group: Longhorn V2 Data Engine (Experimental Feature) Settings type: boolean default: false - variable: defaultSettings.dataEngineHugepageEnabled label: V2 Data Engine description: >- Applies only to the V2 Data Engine. Enables hugepages for the Storage Performance Development Kit (SPDK) target daemon. If disabled, legacy memory is used. Allocation size is set via the Data Engine Memory Size setting. group: Longhorn V2 Data Engine (Experimental Feature) Settings type: string default: '{"v2":"true"}' - variable: defaultSettings.dataEngineMemorySize label: V2 Data Engine description: >- Applies only to the V2 Data Engine. Specifies the memory size, in MiB, allocated to the Storage Performance Development Kit (SPDK) target daemon. When hugepage is enabled, this defines the hugepage size; when legacy memory is used, hugepage is disabled. group: Longhorn V2 Data Engine (Experimental Feature) Settings type: string default: '{"v2":"2048"}' - variable: defaultSettings.dataEngineLogLevel label: Data Engine Log Level description: >- Applies only to the V2 Data Engine. Specifies the log level for the Storage Performance Development Kit (SPDK) target daemon. Supported values are: Error, Warning, Notice, Info, and Debug. The default is Notice. group: Longhorn V2 Data Engine (Experimental Feature) Settings type: string default: '{"v2":"Notice"}' - variable: defaultSettings.dataEngineLogFlags label: Data Engine Log Flags description: >- Applies only to the V2 Data Engine. Specifies the log flags for the Storage Performance Development Kit (SPDK) target daemon. group: Longhorn V2 Data Engine (Experimental Feature) Settings type: string default: '{"v2":""}' - variable: defaultSettings.autoCleanupSnapshotWhenDeleteBackup label: Auto Cleanup Snapshot When Delete Backup description: >- Setting that automatically cleans up the snapshot when the backup is deleted. group: Longhorn Default Settings type: boolean default: false - variable: defaultSettings.autoCleanupSnapshotAfterOnDemandBackupCompleted label: Auto Cleanup Snapshot After On-Demand Backup Completed description: >- Setting that automatically cleans up the snapshot after the on-demand backup is completed. group: Longhorn Default Settings type: boolean default: false - variable: defaultSettings.rwxVolumeFastFailover label: RWX Volume Fast Failover (Experimental) description: >- Setting that allows Longhorn to detect node failure and immediately migrate affected RWX volumes. group: Longhorn Default Settings type: boolean default: false - variable: defaultSettings.offlineReplicaRebuilding label: Offline Replica Rebuilding description: >- Enables automatic rebuilding of degraded replicas while the volume is detached. This setting only takes effect if the individual volume setting is set to `ignored` or `enabled`. group: Longhorn Default Settings type: string default: '{"v1":"false","v2":"false"}' - variable: defaultSettings.dataEngineCPUMask label: Data Engine CPU Mask description: >- Applies only to the V2 Data Engine. Specifies the CPU cores on which the Storage Performance Development Kit (SPDK) target daemon runs. The daemon is deployed in each Instance Manager pod. Ensure that the number of assigned cores does not exceed the guaranteed Instance Manager CPUs for the V2 Data Engine. group: Longhorn Default Settings type: string default: '{"v2":"0x1"}' - variable: defaultSettings.replicaRebuildingBandwidthLimit label: Replica Rebuilding Bandwidth Limit description: >- Applies only to the V2 Data Engine. Specifies the default write bandwidth limit, in megabytes per second (MB/s), for volume replica rebuilding. If this value is set to 0, there will be no write bandwidth limitation. Individual volumes can override this setting by specifying their own rebuilding bandwidth limit. group: Longhorn Default Settings type: string default: '{"v2":"0"}' - variable: defaultSettings.instanceManagerPodLivenessProbeTimeout label: Instance Manager Pod Liveness Probe Timeout description: >- In seconds. The setting specifies the timeout for the instance manager pod liveness probe. The default value is 10 seconds. group: Longhorn Default Settings type: int min: 1 max: 60 default: 10 - variable: defaultSettings.snapshotHeavyTaskConcurrentLimit label: Snapshot Heavy Task Concurrent Limit description: >- Setting that controls how many snapshot heavy task operations (such as purge and clone) can run concurrently per node. This is a best-effort mechanism: due to the distributed nature of the system, temporary oversubscription may occur. The limiter reduces worst-case overload but does not guarantee perfect enforcement. group: Longhorn Default Settings type: int min: 0 default: 5 - variable: defaultSettings.nodeDiskHealthMonitoring label: Node Disk Health Monitoring description: >- Controls whether Longhorn monitors and records health information for node disks. When disabled, disk health checks and status updates are skipped. group: Longhorn Default Settings type: boolean default: true - variable: defaultSettings.csiAllowedTopologyKeys label: CSI Allowed Topology Keys description: >- Comma-separated list of topology keys that the Longhorn CSI driver is allowed to pass through. When empty (default), no topology keys are passed through, and PVs will have no nodeAffinity. When configured (e.g., "topology.kubernetes.io/zone,topology.kubernetes.io/region"), only the specified keys are kept in topology segments. All other keys are filtered out from both CreateVolumeResponse.AccessibleTopology and NodeGetInfo topology. group: Longhorn Default Settings type: string default: null - variable: persistence.defaultClass default: 'true' description: Setting that allows you to specify the default Longhorn StorageClass. label: Default Storage Class group: Longhorn Storage Class Settings required: true type: boolean subquestions: [] - variable: persistence.reclaimPolicy label: Storage Class Retain Policy description: >- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") group: Longhorn Storage Class Settings required: true type: enum options: - Delete - Retain default: Delete subquestions: [] - variable: persistence.volumeBindingMode label: Storage Class Volume Binding Mode description: >- VolumeBindingMode controls when volume binding and dynamic provisioning should occur. (Options: "Immediate", "WaitForFirstConsumer") (Defaults to "Immediate") group: Longhorn Storage Class Settings required: true type: enum options: - Immediate - WaitForFirstConsumer default: Immediate subquestions: [] - variable: persistence.defaultFsType label: Storage Class Filesystem Type description: Filesystem type of the default Longhorn StorageClass group: Longhorn Storage Class Settings type: enum options: - xfs - ext4 default: ext4 subquestions: [] - variable: persistence.disableRevisionCounter label: Default Storage Class Disable Revision Counter description: >- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the volume-head-xxx.img file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. (Options: "true", "false") group: Longhorn Storage Class Settings required: true type: enum options: - 'true' - 'false' default: 'true' subquestions: [] - variable: persistence.defaultClassReplicaCount description: Replica count of the default Longhorn StorageClass. label: Default Storage Class Replica Count group: Longhorn Storage Class Settings type: int min: 1 max: 10 default: 3 subquestions: [] - variable: persistence.defaultDataLocality description: >- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") label: Default Storage Class Data Locality group: Longhorn Storage Class Settings type: enum options: - disabled - best-effort default: disabled subquestions: [] - variable: persistence.recurringJobSelector.enable description: >- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. group: Longhorn Storage Class Settings label: Enable Storage Class Recurring Job Selector type: boolean default: false show_subquestion_if: true subquestions: - variable: persistence.recurringJobSelector.jobList description: >- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) label: Storage Class Recurring Job Selector List group: Longhorn Storage Class Settings type: string default: null - variable: persistence.defaultDiskSelector.enable description: >- Setting that allows you to enable the disk selector for the default Longhorn StorageClass. group: Longhorn Storage Class Settings label: Enable Storage Class Disk Selector type: boolean default: false show_subquestion_if: true subquestions: - variable: persistence.defaultDiskSelector.selector label: Storage Class Disk Selector description: >- Disk selector for the default Longhorn StorageClass. Longhorn uses only disks with the specified tags for storing volume data. (Examples: "nvme,sata") group: Longhorn Storage Class Settings type: string default: null - variable: persistence.defaultNodeSelector.enable description: >- Setting that allows you to enable the node selector for the default Longhorn StorageClass. group: Longhorn Storage Class Settings label: Enable Storage Class Node Selector type: boolean default: false show_subquestion_if: true subquestions: - variable: persistence.defaultNodeSelector.selector label: Storage Class Node Selector description: >- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") group: Longhorn Storage Class Settings type: string default: null - variable: persistence.backingImage.enable description: Setting that allows you to use a backing image in a Longhorn StorageClass. group: Longhorn Storage Class Settings label: Default Storage Class Backing Image type: boolean default: false show_subquestion_if: true subquestions: - variable: persistence.backingImage.name description: >- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. label: Storage Class Backing Image Name group: Longhorn Storage Class Settings type: string default: null - variable: persistence.backingImage.expectedChecksum description: >- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. **Caution** [1] This field is useful only if the backing image name is specified. [2] Specifying a checksum is not recommended when the data source type is \"export-from-volume\". label: Storage Class Backing Image Expected SHA512 Checksum group: Longhorn Storage Class Settings type: string default: null - variable: persistence.backingImage.dataSourceType description: >- Data source type of a backing image used in a Longhorn StorageClass. If the backing image exists in the cluster, Longhorn uses this setting to verify the image. If the backing image does not exist, Longhorn creates one using the specified data source type. **Caution** [1] This field is useful only if the backing image name is specified. [2] Backing images with data source type \"upload\" are best created using the Longhorn UI. Uploading requires sending file data to Longhorn after object creation, which can be complicated when performed manually. label: Storage Class Backing Image Data Source Type group: Longhorn Storage Class Settings type: enum options: - '' - download - upload - export-from-volume default: '' - variable: persistence.backingImage.dataSourceParameters description: >- Data source parameters of a backing image used in a Longhorn StorageClass. You can specify a JSON string of a map. (Example: `'{"url":"https://backing-image-example.s3-region.amazonaws.com/test-backing-image"}'`) **Caution** [1] This field is useful only if the backing image name is specified. [2] Ensure that quotes are used correctly when specifying parameters. label: Storage Class Backing Image Data Source Parameters group: Longhorn Storage Class Settings type: string default: null - variable: persistence.unmapMarkSnapChainRemoved description: >- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") label: Default Storage Class Remove Snapshots During Filesystem Trim group: Longhorn Storage Class Settings type: enum options: - ignored - enabled - disabled default: ignored subquestions: [] - variable: persistence.dataEngine description: >- Setting that allows you to specify the data engine version for the default Longhorn StorageClass. (Options: "v1", "v2") label: Default Storage Class Data Engine group: Longhorn Storage Class Settings type: enum options: - v1 - v2 default: v1 subquestions: [] - variable: persistence.backupTargetName description: >- Setting that allows you to specify the backup target for the default Longhorn StorageClass label: Default Storage Class Backup Target Name group: Longhorn Storage Class Settings type: string default: default subquestions: [] - variable: ingress.enabled default: 'false' description: Expose app using Layer 7 Load Balancer - ingress type: boolean group: Services and Load Balancing label: Expose app using Layer 7 Load Balancer show_subquestion_if: true subquestions: - variable: ingress.host default: xip.io description: Hostname of the Layer 7 load balancer. type: hostname required: true label: Layer 7 Load Balancer Hostname - variable: ingress.path default: / description: >- Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}. type: string required: true label: Ingress Path - variable: ingress.pathType default: ImplementationSpecific description: >- Path type for the ingress. (Options: "ImplementationSpecific", "Exact", "Prefix") type: enum options: - ImplementationSpecific - Exact - Prefix required: true label: Ingress Path Type - variable: httproute.enabled default: 'false' description: Expose app using Gateway API HTTPRoute type: boolean group: Services and Load Balancing label: Expose app using Gateway API HTTPRoute show_subquestion_if: true subquestions: - variable: httproute.parentRefs default: '[]' description: >- Gateway references as JSON array. Required fields: name, namespace. Optional: group (default: gateway.networking.k8s.io), kind (default: Gateway), sectionName. Example: [{"name":"my-gateway","namespace":"default","sectionName":"https"}] type: string required: true label: Gateway References (JSON array) - variable: httproute.hostnames default: '[]' description: >- Hostnames for HTTPRoute as JSON array (e.g., ["longhorn.example.com"]) type: string required: true label: Hostnames (JSON array) - variable: httproute.path default: / description: >- Default path for HTTPRoute. You can access the Longhorn UI by following the full path. type: string required: true label: HTTPRoute Path - variable: httproute.pathType default: PathPrefix description: >- Path match type for HTTPRoute. (Options: "Exact", "PathPrefix") type: enum options: - Exact - PathPrefix required: true label: HTTPRoute Path Type - variable: service.ui.type default: Rancher-Proxy description: >- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") type: enum options: - ClusterIP - NodePort - LoadBalancer - Rancher-Proxy label: Longhorn UI Service show_if: ingress.enabled=false group: Services and Load Balancing show_subquestion_if: NodePort subquestions: - variable: service.ui.nodePort default: '' description: >- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. type: int min: 30000 max: 32767 show_if: service.ui.type=NodePort||service.ui.type=LoadBalancer label: UI Service NodePort number - variable: longhorn.default_resource default: 'false' description: >- Customize the default resource before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn. label: Customize Default Resources type: boolean show_subquestion_if: true group: Longhorn Default Resources subquestions: - variable: defaultBackupStore.backupTarget label: Backup Target description: >- Endpoint used to access the backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") group: Longhorn Default Resources type: string default: null - variable: defaultBackupStore.backupTargetCredentialSecret label: Backup Target Credential Secret description: Name of the Kubernetes secret associated with the backup target. group: Longhorn Default Resources type: string default: null - variable: defaultBackupStore.backupstorePollInterval label: Backupstore Poll Interval description: >- Number of seconds that Longhorn waits before checking the backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. group: Longhorn Default Resources type: int min: 0 default: 300 - variable: enablePSP default: 'false' description: >- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. label: Pod Security Policy type: boolean group: Other Settings subquestions: [] - variable: global.cattle.windowsCluster.enabled default: 'false' description: Setting that allows Longhorn to run on a Rancher Windows cluster. label: Rancher Windows Cluster type: boolean group: Other Settings subquestions: [] - variable: networkPolicies.enabled description: >- Setting that allows you to enable network policies that control access to Longhorn pods. **Caution** The Rancher Proxy will work only if this feature is enabled and a custom NetworkPolicy is added. group: Other Settings label: Network Policies default: 'false' type: boolean subquestions: - variable: networkPolicies.type label: Network Policies for Ingress description: >- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") show_if: networkPolicies.enabled=true&&ingress.enabled=true type: enum default: rke2 options: - rke1 - rke2 - k3s ================================================ FILE: chart/templates/NOTES.txt ================================================ Longhorn is now installed on the cluster! Please wait a few minutes for other Longhorn components such as CSI deployments, Engine Images, and Instance Managers to be initialized. Visit our documentation at https://longhorn.io/docs/ ================================================ FILE: chart/templates/_helpers.tpl ================================================ {{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. */}} {{- define "longhorn.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). */}} {{- define "longhorn.fullname" -}} {{- $name := default .Chart.Name .Values.nameOverride -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{- define "longhorn.managerIP" -}} {{- $fullname := (include "longhorn.fullname" .) -}} {{- printf "http://%s-backend:9500" $fullname | trunc 63 | trimSuffix "-" -}} {{- end -}} {{- define "secret" }} {{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} {{- end }} {{- /* longhorn.labels generates the standard Helm labels. */ -}} {{- define "longhorn.labels" -}} app.kubernetes.io/name: {{ template "longhorn.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} {{- end -}} {{- define "system_default_registry" -}} {{- if .Values.global.cattle.systemDefaultRegistry -}} {{- .Values.global.cattle.systemDefaultRegistry -}} {{- else -}} {{- "" -}} {{- end -}} {{- end -}} {{- define "registry_url" -}} {{- if .Values.privateRegistry.registryUrl -}} {{- .Values.privateRegistry.registryUrl -}} {{- else -}} {{ include "system_default_registry" . }} {{- end -}} {{- end -}} {{- /* define the longhorn release namespace */ -}} {{- define "release_namespace" -}} {{- if .Values.namespaceOverride -}} {{- .Values.namespaceOverride -}} {{- else -}} {{- .Release.Namespace -}} {{- end -}} {{- end -}} {{- /* multiTypeSetting helper Input: any value (string, number, or map) Output: properly quoted YAML string */ -}} {{- define "longhorn.multiTypeSetting" -}} {{- $v := . -}} {{- if kindIs "map" $v -}} {{- $v | toJson | quote -}} {{- else -}} {{- $v | quote -}} {{- end -}} {{- end -}} {{/* Optional timezone injection for all Longhorn workloads. When .Values.global.timezone is set, this snippet renders a TZ env var. */}} {{- define "longhorn.timezoneEnv" -}} {{- if .Values.global.timezone }} - name: TZ value: {{ .Values.global.timezone | quote }} {{- end }} {{- end -}} ================================================ FILE: chart/templates/clusterrole.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-role labels: {{- include "longhorn.labels" . | nindent 4 }} rules: - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - "*" - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "delete", "deletecollection"] - apiGroups: [""] resources: ["secrets", "services", "endpoints", "configmaps", "serviceaccounts", "pods/log"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status", "nodes"] verbs: ["*"] - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "statefulsets", "deployments", "replicasets"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "list", "watch"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["watch", "list"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "volumeattributesclasses", "csinodes", "csidrivers", "csistoragecapacities"] verbs: ["*"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] verbs: ["*"] - apiGroups: ["longhorn.io"] resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status", "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", {{- if .Values.openshift.enabled }} "engineimages/finalizers", "nodes/finalizers", "instancemanagers/finalizers", {{- end }} "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch"] - apiGroups: ["metrics.k8s.io"] resources: ["pods", "nodes"] verbs: ["get", "list"] - apiGroups: ["apiregistration.k8s.io"] resources: ["apiservices"] verbs: ["list", "watch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] verbs: ["get", "list", "create", "patch", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterrolebindings", "clusterroles"] verbs: ["*"] {{- if .Values.openshift.enabled }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-ocp-privileged-role labels: {{- include "longhorn.labels" . | nindent 4 }} rules: - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] resourceNames: ["anyuid", "privileged"] verbs: ["use"] {{- end }} ================================================ FILE: chart/templates/clusterrolebinding.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-bind labels: {{- include "longhorn.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: {{ include "release_namespace" . }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-support-bundle labels: {{- include "longhorn.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: longhorn-support-bundle namespace: {{ include "release_namespace" . }} {{- if .Values.openshift.enabled }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-ocp-privileged-bind labels: {{- include "longhorn.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-ocp-privileged-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: {{ include "release_namespace" . }} - kind: ServiceAccount name: longhorn-ui-service-account namespace: {{ include "release_namespace" . }} - kind: ServiceAccount name: default # supportbundle-agent-support-bundle uses default sa namespace: {{ include "release_namespace" . }} {{- end }} ================================================ FILE: chart/templates/crds.yaml ================================================ # Generated crds.yaml from github.com/longhorn/longhorn-manager/k8s/pkg/apis and the crds.yaml will be copied to longhorn/longhorn chart/templates and cannot be directly used by kubectl apply. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backingimagedatasources.longhorn.io spec: group: longhorn.io names: kind: BackingImageDataSource listKind: BackingImageDataSourceList plural: backingimagedatasources shortNames: - lhbids singular: backingimagedatasource scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID of the provisioned backing image file jsonPath: .spec.uuid name: UUID type: string - description: The current state of the pod used to provision the backing image file from source jsonPath: .status.currentState name: State type: string - description: The data source type jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size jsonPath: .status.size name: Size type: string - description: The node the backing image file will be prepared on jsonPath: .spec.nodeID name: Node type: string - description: The disk the backing image file will be prepared on jsonPath: .spec.diskUUID name: DiskUUID type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageDataSource is where Longhorn stores backing image data source object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source properties: checksum: type: string diskPath: type: string diskUUID: type: string fileTransferred: type: boolean nodeID: type: string parameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string uuid: type: string type: object status: description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source properties: checksum: type: string currentState: type: string ip: type: string message: type: string ownerID: type: string progress: type: integer runningParameters: additionalProperties: type: string nullable: true type: object size: format: int64 type: integer storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backingimagemanagers.longhorn.io spec: group: longhorn.io names: kind: BackingImageManager listKind: BackingImageManagerList plural: backingimagemanagers shortNames: - lhbim singular: backingimagemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The current state of the manager jsonPath: .status.currentState name: State type: string - description: The image the manager pod will use jsonPath: .spec.image name: Image type: string - description: The node the manager is on jsonPath: .spec.nodeID name: Node type: string - description: The disk the manager is responsible for jsonPath: .spec.diskUUID name: DiskUUID type: string - description: The disk path the manager is using jsonPath: .spec.diskPath name: DiskPath type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageManager is where Longhorn stores backing image manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager properties: backingImages: additionalProperties: type: string type: object diskPath: type: string diskUUID: type: string image: type: string nodeID: type: string type: object status: description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImageFileMap: additionalProperties: properties: currentChecksum: type: string message: type: string name: type: string progress: type: integer realSize: format: int64 type: integer senderManagerAddress: type: string sendingReference: type: integer size: format: int64 type: integer state: type: string uuid: type: string virtualSize: format: int64 type: integer type: object nullable: true type: object currentState: type: string ip: type: string ownerID: type: string storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backingimages.longhorn.io spec: group: longhorn.io names: kind: BackingImage listKind: BackingImageList plural: backingimages shortNames: - lhbi singular: backingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID jsonPath: .status.uuid name: UUID type: string - description: The source of the backing image file data jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size in each disk jsonPath: .status.size name: Size type: string - description: The virtual size of the image (may be larger than file size) jsonPath: .status.virtualSize name: VirtualSize type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImage is where Longhorn stores backing image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageSpec defines the desired state of the Longhorn backing image properties: checksum: type: string dataEngine: default: v1 enum: - v1 - v2 type: string diskFileSpecMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string evictionRequested: type: boolean type: object type: object diskSelector: items: type: string type: array disks: additionalProperties: type: string description: Deprecated. We are now using DiskFileSpecMap to assign different spec to the file on different disks. type: object minNumberOfCopies: type: integer nodeSelector: items: type: string type: array secret: type: string secretNamespace: type: string sourceParameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string type: object status: description: BackingImageStatus defines the observed state of the Longhorn backing image status properties: checksum: type: string diskFileStatusMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string lastStateTransitionTime: type: string message: type: string progress: type: integer state: type: string type: object nullable: true type: object diskLastRefAtMap: additionalProperties: type: string nullable: true type: object ownerID: type: string realSize: description: Real size of image in bytes, which may be smaller than the size when the file is a sparse file. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer size: format: int64 type: integer uuid: type: string v2FirstCopyDisk: type: string v2FirstCopyStatus: description: It is pending -> in-progress -> ready/failed type: string virtualSize: description: Virtual size of image in bytes, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backupbackingimages.longhorn.io spec: group: longhorn.io names: kind: BackupBackingImage listKind: BackupBackingImageList plural: backupbackingimages shortNames: - lhbbi singular: backupbackingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The backing image name jsonPath: .status.backingImage name: BackingImage type: string - description: The backing image size jsonPath: .status.size name: Size type: string - description: The backing image backup upload finished time jsonPath: .status.backupCreatedAt name: BackupCreatedAt type: string - description: The backing image backup state jsonPath: .status.state name: State type: string - description: The last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupBackingImage is where Longhorn stores backing image backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupTargetName: description: The backup target name. nullable: true type: string labels: additionalProperties: type: string description: The labels of backing image backup. type: object syncRequestedAt: description: The time to request run sync the remote backing image backup. format: date-time nullable: true type: string userCreated: description: Is this CR created by user through API or UI. type: boolean required: - backingImage - userCreated type: object status: description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupCreatedAt: description: The backing image backup upload finished time. type: string checksum: description: The checksum of the backing image. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the backing image backup. type: string labels: additionalProperties: type: string description: The labels of backing image backup. nullable: true type: object lastSyncedAt: description: The last time that the backing image backup was synced with the remote backup target. format: date-time nullable: true type: string managerAddress: description: The address of the backing image manager that runs backing image backup. type: string messages: additionalProperties: type: string description: The error messages when listing or inspecting backing image backup. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this CR. type: string progress: description: The backing image backup progress. type: integer secret: description: Record the secret if this backup backing image is encrypted type: string secretNamespace: description: Record the secret namespace if this backup backing image is encrypted type: string size: description: The backing image size. format: int64 type: integer state: description: |- The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The backing image backup URL. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backups.longhorn.io spec: group: longhorn.io names: kind: Backup listKind: BackupList plural: backups shortNames: - lhb singular: backup scope: Namespaced versions: - additionalPrinterColumns: - description: The snapshot name jsonPath: .status.snapshotName name: SnapshotName type: string - description: The snapshot size jsonPath: .status.size name: SnapshotSize type: string - description: The snapshot creation time jsonPath: .status.snapshotCreatedAt name: SnapshotCreatedAt type: string - description: The backup target name jsonPath: .status.backupTargetName name: BackupTarget type: string - description: The backup state jsonPath: .status.state name: State type: string - description: The backup last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: Backup is where Longhorn stores backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupSpec defines the desired state of the Longhorn backup properties: backupBlockSize: description: The backup block size. 0 means the legacy default size 2MiB, and -1 indicate the block size is invalid. enum: - "-1" - "2097152" - "16777216" format: int64 type: string backupMode: description: |- The backup mode of this backup. Can be "full" or "incremental" enum: - full - incremental type: string labels: additionalProperties: type: string description: The labels of snapshot backup. type: object snapshotName: description: The snapshot name. type: string syncRequestedAt: description: The time to request run sync the remote backup. format: date-time nullable: true type: string type: object status: description: BackupStatus defines the observed state of the Longhorn backup properties: backupCreatedAt: description: The snapshot backup upload finished time. type: string backupTargetName: description: The backup target name. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the snapshot backup. type: string labels: additionalProperties: type: string description: The labels of snapshot backup. nullable: true type: object lastSyncedAt: description: The last time that the backup was synced with the remote backup target. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when calling longhorn engine on listing or inspecting backups. nullable: true type: object newlyUploadDataSize: description: Size in bytes of newly uploaded data type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup CR. type: string progress: description: The snapshot backup progress. type: integer reUploadedDataSize: description: Size in bytes of reuploaded data type: string replicaAddress: description: The address of the replica that runs snapshot backup. type: string size: description: The snapshot size. type: string snapshotCreatedAt: description: The snapshot creation time. type: string snapshotName: description: The snapshot name. type: string state: description: |- The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The snapshot backup URL. type: string volumeBackingImageName: description: The volume's backing image name. type: string volumeCreated: description: The volume creation time. type: string volumeName: description: The volume name. type: string volumeSize: description: The volume size. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backuptargets.longhorn.io spec: group: longhorn.io names: kind: BackupTarget listKind: BackupTargetList plural: backuptargets shortNames: - lhbt singular: backuptarget scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target URL jsonPath: .spec.backupTargetURL name: URL type: string - description: The backup target credential secret jsonPath: .spec.credentialSecret name: Credential type: string - description: The backup target poll interval jsonPath: .spec.pollInterval name: LastBackupAt type: string - description: Indicate whether the backup target is available or not jsonPath: .status.available name: Available type: boolean - description: The backup target last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupTarget is where Longhorn stores backup target object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupTargetSpec defines the desired state of the Longhorn backup target properties: backupTargetURL: description: The backup target URL. type: string credentialSecret: description: The backup target credential secret. type: string pollInterval: description: The interval that the cluster needs to run sync with the backup target. type: string syncRequestedAt: description: The time to request run sync the remote backup target. format: date-time nullable: true type: string type: object status: description: BackupTargetStatus defines the observed state of the Longhorn backup target properties: available: description: Available indicates if the remote backup target is available or not. type: boolean conditions: description: Records the reason on why the backup target is unavailable. items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array lastSyncedAt: description: The last time that the controller synced with the remote backup target. format: date-time nullable: true type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup target CR. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: backupvolumes.longhorn.io spec: group: longhorn.io names: kind: BackupVolume listKind: BackupVolumeList plural: backupvolumes shortNames: - lhbv singular: backupvolume scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target name jsonPath: .spec.backupTargetName name: BackupTarget type: string - description: The backup volume creation time jsonPath: .status.createdAt name: CreatedAt type: string - description: The backup volume last backup name jsonPath: .status.lastBackupName name: LastBackupName type: string - description: The backup volume last backup time jsonPath: .status.lastBackupAt name: LastBackupAt type: string - description: The backup volume last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupVolume is where Longhorn stores backup volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupVolumeSpec defines the desired state of the Longhorn backup volume properties: backupTargetName: description: The backup target name that the backup volume was synced. nullable: true type: string syncRequestedAt: description: The time to request run sync the remote backup volume. format: date-time nullable: true type: string volumeName: description: The volume name that the backup volume was used to backup. type: string type: object status: description: BackupVolumeStatus defines the observed state of the Longhorn backup volume properties: backingImageChecksum: description: the backing image checksum. type: string backingImageName: description: The backing image name. type: string createdAt: description: The backup volume creation time. type: string dataStored: description: The backup volume block count. type: string labels: additionalProperties: type: string description: The backup volume labels. nullable: true type: object lastBackupAt: description: The latest volume backup time. type: string lastBackupName: description: The latest volume backup name. type: string lastModificationTime: description: The backup volume config last modification time. format: date-time nullable: true type: string lastSyncedAt: description: The last time that the backup volume was synced into the cluster. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when call longhorn engine on list or inspect backup volumes. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this backup volume CR. type: string size: description: The backup volume size. type: string storageClassName: description: the storage class name of pv/pvc binding with the volume. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: engineimages.longhorn.io spec: group: longhorn.io names: kind: EngineImage listKind: EngineImageList plural: engineimages shortNames: - lhei singular: engineimage scope: Namespaced versions: - additionalPrinterColumns: - description: Compatibility of the engine image jsonPath: .status.incompatible name: Incompatible type: boolean - description: State of the engine image jsonPath: .status.state name: State type: string - description: The Longhorn engine image jsonPath: .spec.image name: Image type: string - description: Number of resources using the engine image jsonPath: .status.refCount name: RefCount type: integer - description: The build date of the engine image jsonPath: .status.buildDate name: BuildDate type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: EngineImage is where Longhorn stores engine image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineImageSpec defines the desired state of the Longhorn engine image properties: image: minLength: 1 type: string required: - image type: object status: description: EngineImageStatus defines the observed state of the Longhorn engine image properties: buildDate: type: string cliAPIMinVersion: type: integer cliAPIVersion: type: integer conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array controllerAPIMinVersion: type: integer controllerAPIVersion: type: integer dataFormatMinVersion: type: integer dataFormatVersion: type: integer gitCommit: type: string incompatible: type: boolean noRefSince: type: string nodeDeploymentMap: additionalProperties: type: boolean nullable: true type: object ownerID: type: string refCount: type: integer state: type: string version: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: engines.longhorn.io spec: group: longhorn.io names: kind: Engine listKind: EngineList plural: engines shortNames: - lhe singular: engine scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the engine jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the engine jsonPath: .status.currentState name: State type: string - description: The node that the engine is on jsonPath: .spec.nodeID name: Node type: string - description: The instance manager of the engine jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the engine jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Engine is where Longhorn stores engine object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineSpec defines the desired state of the Longhorn engine properties: active: type: boolean backupVolume: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string disableFrontend: type: boolean frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string logRequested: type: boolean nodeID: type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. maximum: 5 minimum: 0 type: integer replicaAddressMap: additionalProperties: type: string type: object requestedBackupRestore: type: string requestedDataSource: type: string revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemovedEnabled: type: boolean upgradedReplicaAddressMap: additionalProperties: type: string type: object volumeName: type: string volumeSize: format: int64 type: string type: object status: description: EngineStatus defines the observed state of the Longhorn engine properties: backupStatus: additionalProperties: properties: backupURL: type: string error: type: string progress: type: integer replicaAddress: type: string snapshotName: type: string state: type: string type: object nullable: true type: object cloneStatus: additionalProperties: properties: error: type: string fromReplicaAddress: type: string isCloning: type: boolean progress: type: integer snapshotName: type: string state: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentReplicaAddressMap: additionalProperties: type: string nullable: true type: object currentSize: format: int64 type: string currentState: type: string endpoint: type: string instanceManagerName: type: string ip: type: string isExpanding: type: boolean lastExpansionError: type: string lastExpansionFailedAt: type: string lastRestoredBackup: type: string logFetched: type: boolean ownerID: type: string port: type: integer purgeStatus: additionalProperties: properties: error: type: string isPurging: type: boolean progress: type: integer state: type: string type: object nullable: true type: object rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. minimum: 0 type: integer rebuildStatus: additionalProperties: properties: appliedRebuildingMBps: format: int64 type: integer error: type: string fromReplicaAddress: description: Deprecated. We are now using FromReplicaAddressList to list all source replicas. type: string fromReplicaAddressList: items: type: string type: array isRebuilding: type: boolean progress: type: integer state: type: string type: object nullable: true type: object replicaModeMap: additionalProperties: type: string nullable: true type: object replicaTransitionTimeMap: additionalProperties: type: string description: |- ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). type: object restoreStatus: additionalProperties: properties: backupURL: type: string currentRestoringBackup: type: string error: type: string filename: type: string isRestoring: type: boolean lastRestored: type: string progress: type: integer state: type: string type: object nullable: true type: object salvageExecuted: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string snapshots: additionalProperties: properties: children: additionalProperties: type: boolean nullable: true type: object created: type: string labels: additionalProperties: type: string nullable: true type: object name: type: string parent: type: string removed: type: boolean size: type: string usercreated: type: boolean type: object nullable: true type: object snapshotsError: type: string started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer unmapMarkSnapChainRemovedEnabled: type: boolean uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: instancemanagers.longhorn.io spec: group: longhorn.io names: kind: InstanceManager listKind: InstanceManagerList plural: instancemanagers shortNames: - lhim singular: instancemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the instance manager jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the instance manager jsonPath: .status.currentState name: State type: string - description: The type of the instance manager (engine or replica) jsonPath: .spec.type name: Type type: string - description: The node that the instance manager is running on jsonPath: .spec.nodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: InstanceManager is where Longhorn stores instance manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: InstanceManagerSpec defines the desired state of the Longhorn instance manager properties: dataEngine: type: string dataEngineSpec: properties: v2: properties: cpuMask: type: string type: object type: object image: type: string nodeID: type: string type: enum: - aio - engine - replica type: string type: object status: description: InstanceManagerStatus defines the observed state of the Longhorn instance manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImages: additionalProperties: properties: currentChecksum: type: string diskUUID: type: string message: type: string name: type: string progress: type: integer size: format: int64 type: integer state: type: string uuid: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentState: type: string dataEngineStatus: properties: v2: properties: cpuMask: type: string interruptModeEnabled: description: |- InterruptModeEnabled indicates whether the V2 data engine is running in interrupt mode (true) or polling mode (false). Set by Longhorn manager; read-only to users. enum: - "" - "true" - "false" type: string type: object type: object instanceEngines: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object instanceReplicas: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object ip: type: string ownerID: type: string proxyApiMinVersion: type: integer proxyApiVersion: type: integer type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: nodes.longhorn.io spec: group: longhorn.io names: kind: Node listKind: NodeList plural: nodes shortNames: - lhn singular: node scope: Namespaced versions: - additionalPrinterColumns: - description: Indicate whether the node is ready jsonPath: .status.conditions[?(@.type=='Ready')].status name: Ready type: string - description: Indicate whether the user disabled/enabled replica scheduling for the node jsonPath: .spec.allowScheduling name: AllowScheduling type: boolean - description: Indicate whether Longhorn can schedule replicas on the node jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Schedulable type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Node is where Longhorn stores Longhorn node object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: NodeSpec defines the desired state of the Longhorn node properties: allowScheduling: type: boolean disks: additionalProperties: properties: allowScheduling: type: boolean diskDriver: enum: - "" - auto - aio - nvme type: string diskType: enum: - filesystem - block type: string evictionRequested: type: boolean path: type: string storageReserved: format: int64 type: integer tags: items: type: string type: array type: object type: object evictionRequested: type: boolean instanceManagerCPURequest: type: integer name: type: string tags: items: type: string type: array type: object status: description: NodeStatus defines the observed state of the Longhorn node properties: autoEvicting: type: boolean conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskStatus: additionalProperties: properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskDriver: type: string diskName: type: string diskPath: type: string diskType: type: string diskUUID: type: string filesystemType: type: string healthData: additionalProperties: properties: attributes: items: properties: id: type: integer name: type: string rawString: type: string rawValue: format: int64 type: integer threshold: type: integer value: type: integer whenFailed: type: string worst: type: integer type: object type: array capacity: format: int64 type: integer diskName: type: string diskType: type: string firmwareVersion: type: string healthStatus: enum: - FAILED - PASSED - UNKNOWN - WARNING type: string modelName: type: string serialNumber: type: string source: enum: - SMART - SPDK type: string temperature: type: integer type: object type: object healthDataLastCollectedAt: format: date-time type: string instanceManagerName: type: string scheduledBackingImage: additionalProperties: format: int64 type: integer nullable: true type: object scheduledReplica: additionalProperties: format: int64 type: integer nullable: true type: object storageAvailable: format: int64 type: integer storageMaximum: format: int64 type: integer storageScheduled: format: int64 type: integer type: object nullable: true type: object region: type: string snapshotCheckStatus: properties: lastPeriodicCheckedAt: format: date-time type: string type: object zone: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: orphans.longhorn.io spec: group: longhorn.io names: kind: Orphan listKind: OrphanList plural: orphans shortNames: - lho singular: orphan scope: Namespaced versions: - additionalPrinterColumns: - description: The type of the orphan jsonPath: .spec.orphanType name: Type type: string - description: The node that the orphan is on jsonPath: .spec.nodeID name: Node type: string name: v1beta2 schema: openAPIV3Schema: description: Orphan is where Longhorn stores orphan object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: OrphanSpec defines the desired state of the Longhorn orphaned data properties: dataEngine: description: |- The type of data engine for instance orphan. Can be "v1", "v2". enum: - v1 - v2 type: string nodeID: description: The node ID on which the controller is responsible to reconcile this orphan CR. type: string orphanType: description: |- The type of the orphaned data. Can be "replica". type: string parameters: additionalProperties: type: string description: The parameters of the orphaned data type: object type: object status: description: OrphanStatus defines the observed state of the Longhorn orphaned data properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: recurringjobs.longhorn.io spec: group: longhorn.io names: kind: RecurringJob listKind: RecurringJobList plural: recurringjobs shortNames: - lhrj singular: recurringjob scope: Namespaced versions: - additionalPrinterColumns: - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume jsonPath: .spec.groups name: Groups type: string - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup" jsonPath: .spec.task name: Task type: string - description: The cron expression represents recurring job scheduling jsonPath: .spec.cron name: Cron type: string - description: The number of snapshots/backups to keep for the volume jsonPath: .spec.retain name: Retain type: integer - description: The concurrent job to run by each cron job jsonPath: .spec.concurrency name: Concurrency type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date - description: Specify the labels jsonPath: .spec.labels name: Labels type: string name: v1beta2 schema: openAPIV3Schema: description: RecurringJob is where Longhorn stores recurring job object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: RecurringJobSpec defines the desired state of the Longhorn recurring job properties: concurrency: description: The concurrency of taking the snapshot/backup. type: integer cron: description: The cron setting. type: string groups: description: The recurring job group. items: type: string type: array labels: additionalProperties: type: string description: The label of the snapshot/backup. type: object name: description: The recurring job name. type: string parameters: additionalProperties: type: string description: |- The parameters of the snapshot/backup. Support parameters: "full-backup-interval", "volume-backup-policy". type: object retain: description: The retain count of the snapshot/backup. type: integer task: description: |- The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup". enum: - snapshot - snapshot-force-create - snapshot-cleanup - snapshot-delete - backup - backup-force-create - filesystem-trim - system-backup type: string type: object status: description: RecurringJobStatus defines the observed state of the Longhorn recurring job properties: executionCount: description: The number of jobs that have been triggered. type: integer ownerID: description: The owner ID which is responsible to reconcile this recurring job CR. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: replicas.longhorn.io spec: group: longhorn.io names: kind: Replica listKind: ReplicaList plural: replicas shortNames: - lhr singular: replica scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the replica jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the replica jsonPath: .status.currentState name: State type: string - description: The node that the replica is on jsonPath: .spec.nodeID name: Node type: string - description: The disk that the replica is on jsonPath: .spec.diskID name: Disk type: string - description: The instance manager of the replica jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the replica jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Replica is where Longhorn stores replica object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ReplicaSpec defines the desired state of the Longhorn replica properties: active: type: boolean backingImage: type: string dataDirectoryName: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string diskID: type: string diskPath: type: string engineName: type: string evictionRequested: type: boolean failedAt: description: |- FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string hardNodeAffinity: type: string healthyAt: description: |- HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string image: type: string lastFailedAt: description: |- LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string lastHealthyAt: description: |- LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string logRequested: type: boolean migrationEngineName: description: |- MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only used for live migration of v2 data engine type: string nodeID: type: string rebuildRetryCount: type: integer revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string unmapMarkDiskChainRemovedEnabled: type: boolean volumeName: type: string volumeSize: format: int64 type: string type: object status: description: ReplicaStatus defines the observed state of the Longhorn replica properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentState: type: string instanceManagerName: type: string ip: type: string logFetched: type: boolean ownerID: type: string port: type: integer salvageExecuted: type: boolean started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: settings.longhorn.io spec: group: longhorn.io names: kind: Setting listKind: SettingList plural: settings shortNames: - lhs singular: setting scope: Namespaced versions: - additionalPrinterColumns: - description: The value of the setting jsonPath: .value name: Value type: string - description: The setting is applied jsonPath: .status.applied name: Applied type: boolean - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Setting is where Longhorn stores setting object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: description: The status of the setting. properties: applied: description: The setting is applied. type: boolean required: - applied type: object value: description: |- The value of the setting. - It can be a non-JSON formatted string that is applied to all the applicable data engines listed in the setting definition. - It can be a JSON formatted string that contains values for applicable data engines listed in the setting definition's Default. type: string required: - value type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: sharemanagers.longhorn.io spec: group: longhorn.io names: kind: ShareManager listKind: ShareManagerList plural: sharemanagers shortNames: - lhsm singular: sharemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the share manager jsonPath: .status.state name: State type: string - description: The node that the share manager is owned by jsonPath: .status.ownerID name: Node type: string - description: The current image of the share manager jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: ShareManager is where Longhorn stores share manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ShareManagerSpec defines the desired state of the Longhorn share manager properties: image: description: Share manager image used for creating a share manager pod type: string type: object status: description: ShareManagerStatus defines the observed state of the Longhorn share manager properties: currentImage: description: The image currently used by the share manager pod type: string endpoint: description: NFS endpoint that can access the mounted filesystem of the volume type: string ownerID: description: The node ID on which the controller is responsible to reconcile this share manager resource type: string state: description: The state of the share manager resource type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: snapshots.longhorn.io spec: group: longhorn.io names: kind: Snapshot listKind: SnapshotList plural: snapshots shortNames: - lhsnap singular: snapshot scope: Namespaced versions: - additionalPrinterColumns: - description: The volume that this snapshot belongs to jsonPath: .spec.volume name: Volume type: string - description: Timestamp when the point-in-time snapshot was taken jsonPath: .status.creationTime name: CreationTime type: string - description: Indicates if the snapshot is ready to be used to restore/backup a volume jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Represents the minimum size of volume required to rehydrate from this snapshot jsonPath: .status.restoreSize name: RestoreSize type: string - description: The actual size of the snapshot jsonPath: .status.size name: Size type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Snapshot is the Schema for the snapshots API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SnapshotSpec defines the desired state of Longhorn Snapshot properties: createSnapshot: description: require creating a new snapshot type: boolean labels: additionalProperties: type: string description: The labels of snapshot nullable: true type: object volume: description: |- the volume that this snapshot belongs to. This field is immutable after creation. type: string required: - volume type: object status: description: SnapshotStatus defines the observed state of Longhorn Snapshot properties: checksum: type: string checksumCalculatedAt: description: |- ChecksumCalculatedAt is the RFC3339 timestamp indicating when the checksum for this snapshot was last calculated or updated. type: string children: additionalProperties: type: boolean nullable: true type: object creationTime: type: string error: type: string labels: additionalProperties: type: string nullable: true type: object markRemoved: type: boolean ownerID: type: string parent: type: string readyToUse: type: boolean restoreSize: format: int64 type: integer size: format: int64 type: integer userCreated: type: boolean type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: supportbundles.longhorn.io spec: group: longhorn.io names: kind: SupportBundle listKind: SupportBundleList plural: supportbundles shortNames: - lhbundle singular: supportbundle scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the support bundle jsonPath: .status.state name: State type: string - description: The issue URL jsonPath: .spec.issueURL name: Issue type: string - description: A brief description of the issue jsonPath: .spec.description name: Description type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SupportBundle is where Longhorn stores support bundle object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle properties: description: description: A brief description of the issue type: string issueURL: description: The issue URL nullable: true type: string nodeID: description: The preferred responsible controller node ID. type: string required: - description type: object status: description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object type: array filename: type: string filesize: format: int64 type: integer image: description: The support bundle manager image type: string managerIP: description: The support bundle manager IP type: string ownerID: description: The current responsible controller node ID type: string progress: type: integer state: type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: systembackups.longhorn.io spec: group: longhorn.io names: kind: SystemBackup listKind: SystemBackupList plural: systembackups shortNames: - lhsb singular: systembackup scope: Namespaced versions: - additionalPrinterColumns: - description: The system backup Longhorn version jsonPath: .status.version name: Version type: string - description: The system backup state jsonPath: .status.state name: State type: string - description: The system backup creation time jsonPath: .status.createdAt name: Created type: string - description: The last time that the system backup was synced into the cluster jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: SystemBackup is where Longhorn stores system backup object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup properties: volumeBackupPolicy: description: |- The create volume backup policy Can be "if-not-present", "always" or "disabled" nullable: true type: string type: object status: description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array createdAt: description: The system backup creation time. format: date-time type: string gitCommit: description: The saved Longhorn manager git commit. nullable: true type: string lastSyncedAt: description: The last time that the system backup was synced into the cluster. format: date-time nullable: true type: string managerImage: description: The saved manager image. type: string ownerID: description: The node ID of the responsible controller to reconcile this SystemBackup. type: string state: description: The system backup state. type: string version: description: The saved Longhorn version. nullable: true type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: systemrestores.longhorn.io spec: group: longhorn.io names: kind: SystemRestore listKind: SystemRestoreList plural: systemrestores shortNames: - lhsr singular: systemrestore scope: Namespaced versions: - additionalPrinterColumns: - description: The system restore state jsonPath: .status.state name: State type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SystemRestore is where Longhorn stores system restore object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore properties: systemBackup: description: The system backup name in the object store. type: string required: - systemBackup type: object status: description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: description: The node ID of the responsible controller to reconcile this SystemRestore. type: string sourceURL: description: The source system backup URL. type: string state: description: The system restore state. type: string type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: volumeattachments.longhorn.io spec: group: longhorn.io names: kind: VolumeAttachment listKind: VolumeAttachmentList plural: volumeattachments shortNames: - lhva singular: volumeattachment scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: VolumeAttachment stores attachment information of a Longhorn volume properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment properties: attachmentTickets: additionalProperties: properties: generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string nodeID: description: The node that this attachment is requesting type: string parameters: additionalProperties: type: string description: Optional additional parameter for this attachment type: object type: type: string type: object type: object volume: description: The name of Longhorn volume of this VolumeAttachment type: string required: - volume type: object status: description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment properties: attachmentTicketStatuses: additionalProperties: properties: conditions: description: Record any error when trying to fulfill this attachment items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string satisfied: description: Indicate whether this attachment ticket has been satisfied type: boolean required: - conditions - satisfied type: object type: object type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: {{- include "longhorn.labels" . | nindent 4 }} longhorn-manager: "" name: volumes.longhorn.io spec: group: longhorn.io names: kind: Volume listKind: VolumeList plural: volumes shortNames: - lhv singular: volume scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the volume jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the volume jsonPath: .status.state name: State type: string - description: The robustness of the volume jsonPath: .status.robustness name: Robustness type: string - description: The scheduled condition of the volume jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Scheduled type: string - description: The size of the volume jsonPath: .spec.size name: Size type: string - description: The node that the volume is currently attaching to jsonPath: .status.currentNodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Volume is where Longhorn stores volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeSpec defines the desired state of the Longhorn volume properties: Standby: type: boolean accessMode: enum: - rwo - rwop - rwx type: string backingImage: type: string x-kubernetes-validations: - message: BackingImage is immutable rule: self == oldSelf backupBlockSize: description: BackupBlockSize indicate the block size to create backups. The block size is immutable. enum: - "2097152" - "16777216" format: int64 type: string backupCompressionMethod: enum: - none - lz4 - gzip type: string backupTargetName: description: The backup target name that the volume will be backed up to or is synced. type: string cloneMode: enum: - "" - full-copy - linked-clone type: string dataEngine: enum: - v1 - v2 type: string dataLocality: enum: - disabled - best-effort - strict-local type: string dataSource: type: string disableFrontend: type: boolean diskSelector: items: type: string type: array encrypted: type: boolean x-kubernetes-validations: - message: Encrypted is immutable rule: self == oldSelf freezeFilesystemForSnapshot: description: Setting that freezes the filesystem on the root partition before a snapshot is created. enum: - ignored - enabled - disabled type: string fromBackup: type: string frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string lastAttachedBy: type: string migratable: type: boolean migrationNodeID: type: string nodeID: type: string nodeSelector: items: type: string type: array numberOfReplicas: type: integer offlineRebuilding: description: |- Specifies whether Longhorn should rebuild replicas while the detached volume is degraded. - ignored: Use the global setting for offline replica rebuilding. - enabled: Enable offline rebuilding for this volume, regardless of the global setting. - disabled: Disable offline rebuilding for this volume, regardless of the global setting enum: - ignored - disabled - enabled type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. When set to 0, it means following the global setting. maximum: 5 minimum: 0 type: integer replicaAutoBalance: enum: - ignored - disabled - least-effort - best-effort type: string replicaDiskSoftAntiAffinity: description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. enum: - ignored - enabled - disabled type: string replicaRebuildingBandwidthLimit: description: ReplicaRebuildingBandwidthLimit controls the maximum write bandwidth (in megabytes per second) allowed on the destination replica during the rebuilding process. Set this value to 0 to disable bandwidth limiting. format: int64 minimum: 0 type: integer replicaSoftAntiAffinity: description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. enum: - ignored - enabled - disabled type: string replicaZoneSoftAntiAffinity: description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. enum: - ignored - enabled - disabled type: string restoreVolumeRecurringJob: enum: - ignored - enabled - disabled type: string revisionCounterDisabled: type: boolean size: format: int64 type: string snapshotDataIntegrity: enum: - ignored - disabled - enabled - fast-check type: string snapshotHashingRequestedAt: description: |- SnapshotHashingRequestedAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when an on-demand snapshot checksum calculation is requested. When this value is set and is later than LastOnDemandSnapshotHashingCompleteAt, the system will calculate checksums for all user snapshots. If SnapshotHashingRequestedAt differs from LastOnDemandSnapshotHashingCompleteAt, it indicates that a hashing request is still in progress, and a new request will be rejected. type: string snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string staleReplicaTimeout: type: integer ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemoved: enum: - ignored - disabled - enabled type: string type: object status: description: VolumeStatus defines the observed state of the Longhorn volume properties: actualSize: format: int64 type: integer cloneStatus: properties: attemptCount: type: integer nextAllowedAttemptAt: type: string snapshot: type: string sourceVolume: type: string state: type: string type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentMigrationNodeID: description: the node that this volume is currently migrating to type: string currentNodeID: type: string expansionRequired: type: boolean frontendDisabled: type: boolean isStandby: type: boolean kubernetesStatus: properties: lastPVCRefAt: type: string lastPodRefAt: type: string namespace: description: determine if PVC/Namespace is history or not type: string pvName: type: string pvStatus: type: string pvcName: type: string workloadsStatus: description: determine if Pod/Workload is history or not items: properties: podName: type: string podStatus: type: string workloadName: type: string workloadType: type: string type: object nullable: true type: array type: object lastBackup: type: string lastBackupAt: type: string lastDegradedAt: type: string lastOnDemandSnapshotHashingCompleteAt: description: |- LastOnDemandSnapshotHashingCompleteAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when the most recent on-demand snapshot checksum calculation completed. When this value matches SnapshotHashingRequestedAt, the requested on-demand checksum calculation is considered complete. type: string ownerID: type: string remountRequestedAt: type: string restoreInitiated: type: boolean restoreRequired: type: boolean robustness: type: string shareEndpoint: type: string shareState: type: string state: type: string type: object type: object served: true storage: true subresources: status: {} ================================================ FILE: chart/templates/daemonset-sa.yaml ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-manager name: longhorn-manager namespace: {{ include "release_namespace" . }} spec: selector: matchLabels: app: longhorn-manager template: metadata: labels: {{- include "longhorn.labels" . | nindent 8 }} app: longhorn-manager {{- with .Values.annotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} spec: containers: - name: longhorn-manager image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: privileged: true command: - longhorn-manager - -d {{- if eq .Values.longhornManager.log.format "json" }} - -j {{- end }} - daemon - --engine-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.engine.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}" - --instance-manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.instanceManager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.instanceManager.repository }}:{{ .Values.image.longhorn.instanceManager.tag }}" - --share-manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.shareManager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }}" - --backing-image-manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.backingImageManager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.backingImageManager.repository }}:{{ .Values.image.longhorn.backingImageManager.tag }}" - --support-bundle-manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.supportBundleKit.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.supportBundleKit.repository }}:{{ .Values.image.longhorn.supportBundleKit.tag }}" - --manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" - --service-account - longhorn-service-account {{- if .Values.preUpgradeChecker.upgradeVersionCheck}} - --upgrade-version-check {{- end }} ports: - containerPort: 9500 name: manager - containerPort: 9502 name: admission-wh - containerPort: 9503 name: recov-backend readinessProbe: httpGet: path: /v1/healthz port: 9502 scheme: HTTPS volumeMounts: - name: boot mountPath: /host/boot/ readOnly: true - name: dev mountPath: /host/dev/ - name: proc mountPath: /host/proc/ readOnly: true - name: etc mountPath: /host/etc/ readOnly: true - name: longhorn mountPath: /var/lib/longhorn/ mountPropagation: Bidirectional - name: longhorn-grpc-tls mountPath: /tls-files/ {{- if .Values.enableGoCoverDir }} - name: go-cover-dir mountPath: /go-cover-dir/ {{- end }} env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName {{- if .Values.enableGoCoverDir }} - name: GOCOVERDIR value: /go-cover-dir/ {{- end }} {{- include "longhorn.timezoneEnv" . | nindent 8 }} {{- with .Values.longhornManager.resources }} resources: {{- toYaml . | nindent 10 }} {{- end }} - name: pre-pull-share-manager-image imagePullPolicy: {{ .Values.image.pullPolicy }} image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.shareManager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }} command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"] volumes: - name: boot hostPath: path: /boot/ - name: dev hostPath: path: /dev/ - name: proc hostPath: path: /proc/ - name: etc hostPath: path: /etc/ - name: longhorn hostPath: path: /var/lib/longhorn/ {{- if .Values.enableGoCoverDir }} - name: go-cover-dir hostPath: path: /go-cover-dir/ type: DirectoryOrCreate {{- end }} - name: longhorn-grpc-tls secret: secretName: longhorn-grpc-tls optional: true {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.longhornManager.priorityClass }} priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornManager.tolerations }} {{ default .Values.global.tolerations .Values.longhornManager.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornManager.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} serviceAccountName: longhorn-service-account updateStrategy: {{ toYaml .Values.longhornManager.updateStrategy | indent 4 }} --- apiVersion: v1 kind: Service metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-manager {{- with .Values.longhornManager.serviceLabels }} {{- toYaml . | nindent 4 }} {{- end }} name: longhorn-backend namespace: {{ include "release_namespace" . }} {{- if .Values.longhornManager.serviceAnnotations }} annotations: {{ toYaml .Values.longhornManager.serviceAnnotations | indent 4 }} {{- end }} spec: type: {{ .Values.service.manager.type }} selector: app: longhorn-manager ports: - name: manager port: 9500 targetPort: manager {{- if .Values.service.manager.nodePort }} nodePort: {{ .Values.service.manager.nodePort }} {{- end }} ================================================ FILE: chart/templates/default-resource.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-resource namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} data: default-resource.yaml: |- {{- if not (kindIs "invalid" .Values.defaultBackupStore.backupTarget) }} backup-target: {{ .Values.defaultBackupStore.backupTarget }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultBackupStore.backupTargetCredentialSecret) }} backup-target-credential-secret: {{ .Values.defaultBackupStore.backupTargetCredentialSecret }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultBackupStore.pollInterval) }} backupstore-poll-interval: {{ .Values.defaultBackupStore.pollInterval }} {{- end }} ================================================ FILE: chart/templates/default-setting.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-setting namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} data: default-setting.yaml: |- {{- if not (kindIs "invalid" .Values.defaultSettings.allowRecurringJobWhileVolumeDetached) }} allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.createDefaultDiskLabeledNodes) }} create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.defaultDataPath) }} default-data-path: {{ .Values.defaultSettings.defaultDataPath | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaSoftAntiAffinity) }} replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaAutoBalance) }} replica-auto-balance: {{ .Values.defaultSettings.replicaAutoBalance | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.storageOverProvisioningPercentage) }} storage-over-provisioning-percentage: {{ .Values.defaultSettings.storageOverProvisioningPercentage | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.storageMinimalAvailablePercentage) }} storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.storageReservedPercentageForDefaultDisk) }} storage-reserved-percentage-for-default-disk: {{ .Values.defaultSettings.storageReservedPercentageForDefaultDisk | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.upgradeChecker) }} upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.upgradeResponderURL) }} upgrade-responder-url: {{ .Values.defaultSettings.upgradeResponderURL | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.managerUrl) }} manager-url: {{ .Values.defaultSettings.managerUrl | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.defaultReplicaCount) }} default-replica-count: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.defaultReplicaCount }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.defaultDataLocality) }} default-data-locality: {{ .Values.defaultSettings.defaultDataLocality | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.defaultLonghornStaticStorageClass) }} default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.failedBackupTTL) }} failed-backup-ttl: {{ .Values.defaultSettings.failedBackupTTL | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.backupExecutionTimeout) }} backup-execution-timeout: {{ .Values.defaultSettings.backupExecutionTimeout | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.restoreVolumeRecurringJobs) }} restore-volume-recurring-jobs: {{ .Values.defaultSettings.restoreVolumeRecurringJobs }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit) }} recurring-successful-jobs-history-limit: {{ .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.recurringJobMaxRetention) }} recurring-job-max-retention: {{ .Values.defaultSettings.recurringJobMaxRetention | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.recurringFailedJobsHistoryLimit) }} recurring-failed-jobs-history-limit: {{ .Values.defaultSettings.recurringFailedJobsHistoryLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.supportBundleFailedHistoryLimit) }} support-bundle-failed-history-limit: {{ .Values.defaultSettings.supportBundleFailedHistoryLimit | quote }} {{- end }} {{- if or (not (kindIs "invalid" .Values.defaultSettings.taintToleration)) (.Values.global.cattle.windowsCluster.enabled) }} taint-toleration: {{ $windowsDefaultSettingTaintToleration := list }}{{ $defaultSettingTaintToleration := list -}} {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} {{- $windowsDefaultSettingTaintToleration = .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} {{- end -}} {{- if not (kindIs "invalid" .Values.defaultSettings.taintToleration) -}} {{- $defaultSettingTaintToleration = .Values.defaultSettings.taintToleration -}} {{- end -}} {{- $taintToleration := list $windowsDefaultSettingTaintToleration $defaultSettingTaintToleration }}{{ join ";" (compact $taintToleration) | quote -}} {{- end }} {{- if or (not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector)) (.Values.global.cattle.windowsCluster.enabled) }} system-managed-components-node-selector: {{ $windowsDefaultSettingNodeSelector := list }}{{ $defaultSettingNodeSelector := list -}} {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} {{ $windowsDefaultSettingNodeSelector = .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} {{- end -}} {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector) -}} {{- $defaultSettingNodeSelector = .Values.defaultSettings.systemManagedComponentsNodeSelector -}} {{- end -}} {{- $nodeSelector := list $windowsDefaultSettingNodeSelector $defaultSettingNodeSelector }}{{ join ";" (compact $nodeSelector) | quote -}} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedCSIComponentsResourceLimits) }} system-managed-csi-components-resource-limits: {{ .Values.defaultSettings.systemManagedCSIComponentsResourceLimits | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.priorityClass) }} priority-class: {{ .Values.defaultSettings.priorityClass | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoSalvage) }} auto-salvage: {{ .Values.defaultSettings.autoSalvage }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly) }} auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.blacklistForAutoDeletePodWhenVolumeDetachedUnexpectedly) }} blacklist-for-auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.blacklistForAutoDeletePodWhenVolumeDetachedUnexpectedly }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.disableSchedulingOnCordonedNode) }} disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaZoneSoftAntiAffinity) }} replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaDiskSoftAntiAffinity) }} replica-disk-soft-anti-affinity: {{ .Values.defaultSettings.replicaDiskSoftAntiAffinity }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.nodeDownPodDeletionPolicy) }} node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.nodeDrainPolicy) }} node-drain-policy: {{ .Values.defaultSettings.nodeDrainPolicy | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.detachManuallyAttachedVolumesWhenCordoned) }} detach-manually-attached-volumes-when-cordoned: {{ .Values.defaultSettings.detachManuallyAttachedVolumesWhenCordoned }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaReplenishmentWaitInterval) }} replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit) }} concurrent-replica-rebuild-per-node-limit: {{ .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.rebuildConcurrentSyncLimit) }} rebuild-concurrent-sync-limit: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.rebuildConcurrentSyncLimit }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit) }} concurrent-volume-backup-restore-per-node-limit: {{ .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.disableRevisionCounter) }} disable-revision-counter: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.disableRevisionCounter }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedPodsImagePullPolicy) }} system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability) }} allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot) }} auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupRecurringJobBackupSnapshot) }} auto-cleanup-recurring-job-backup-snapshot: {{ .Values.defaultSettings.autoCleanupRecurringJobBackupSnapshot }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit) }} concurrent-automatic-engine-upgrade-per-node-limit: {{ .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.backingImageCleanupWaitInterval) }} backing-image-cleanup-wait-interval: {{ .Values.defaultSettings.backingImageCleanupWaitInterval | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.backingImageRecoveryWaitInterval) }} backing-image-recovery-wait-interval: {{ .Values.defaultSettings.backingImageRecoveryWaitInterval | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.guaranteedInstanceManagerCPU) }} guaranteed-instance-manager-cpu: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.guaranteedInstanceManagerCPU }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.kubernetesClusterAutoscalerEnabled) }} kubernetes-cluster-autoscaler-enabled: {{ .Values.defaultSettings.kubernetesClusterAutoscalerEnabled }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.orphanResourceAutoDeletion) }} orphan-resource-auto-deletion: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.orphanResourceAutoDeletion }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.orphanResourceAutoDeletionGracePeriod) }} orphan-resource-auto-deletion-grace-period: {{ .Values.defaultSettings.orphanResourceAutoDeletionGracePeriod | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.storageNetwork) }} storage-network: {{ .Values.defaultSettings.storageNetwork | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.endpointNetworkForRWXVolume) }} endpoint-network-for-rwx-volume: {{ .Values.defaultSettings.endpointNetworkForRWXVolume | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.deletingConfirmationFlag) }} deleting-confirmation-flag: {{ .Values.defaultSettings.deletingConfirmationFlag }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.engineReplicaTimeout) }} engine-replica-timeout: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.engineReplicaTimeout }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrity) }} snapshot-data-integrity: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.snapshotDataIntegrity }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation) }} snapshot-data-integrity-immediate-check-after-snapshot-creation: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityCronjob) }} snapshot-data-integrity-cronjob: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.snapshotDataIntegrityCronjob }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim) }} remove-snapshots-during-filesystem-trim: {{ .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.fastReplicaRebuildEnabled) }} fast-replica-rebuild-enabled: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.fastReplicaRebuildEnabled }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaFileSyncHttpClientTimeout) }} replica-file-sync-http-client-timeout: {{ .Values.defaultSettings.replicaFileSyncHttpClientTimeout | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.longGRPCTimeOut) }} long-grpc-timeout: {{ .Values.defaultSettings.longGRPCTimeOut | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.logLevel) }} log-level: {{ .Values.defaultSettings.logLevel | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.logPath) }} log-path: {{ .Values.defaultSettings.logPath | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.backupCompressionMethod) }} backup-compression-method: {{ .Values.defaultSettings.backupCompressionMethod | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.backupConcurrentLimit) }} backup-concurrent-limit: {{ .Values.defaultSettings.backupConcurrentLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.restoreConcurrentLimit) }} restore-concurrent-limit: {{ .Values.defaultSettings.restoreConcurrentLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.defaultBackupBlockSize) }} default-backup-block-size: {{ .Values.defaultSettings.defaultBackupBlockSize | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.v1DataEngine) }} v1-data-engine: {{ .Values.defaultSettings.v1DataEngine }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.v2DataEngine) }} v2-data-engine: {{ .Values.defaultSettings.v2DataEngine }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.dataEngineHugepageEnabled) }} data-engine-hugepage-enabled: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.dataEngineHugepageEnabled }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.dataEngineMemorySize) }} data-engine-memory-size: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.dataEngineMemorySize }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.allowEmptyNodeSelectorVolume) }} allow-empty-node-selector-volume: {{ .Values.defaultSettings.allowEmptyNodeSelectorVolume }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.allowEmptyDiskSelectorVolume) }} allow-empty-disk-selector-volume: {{ .Values.defaultSettings.allowEmptyDiskSelectorVolume }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.allowCollectingLonghornUsageMetrics) }} allow-collecting-longhorn-usage-metrics: {{ .Values.defaultSettings.allowCollectingLonghornUsageMetrics }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.disableSnapshotPurge) }} disable-snapshot-purge: {{ .Values.defaultSettings.disableSnapshotPurge }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotMaxCount) }} snapshot-max-count: {{ .Values.defaultSettings.snapshotMaxCount | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.dataEngineLogLevel) }} data-engine-log-level: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.dataEngineLogLevel }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.dataEngineLogFlags) }} data-engine-log-flags: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.dataEngineLogFlags }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.freezeFilesystemForSnapshot) }} freeze-filesystem-for-snapshot: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.freezeFilesystemForSnapshot }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSnapshotWhenDeleteBackup) }} auto-cleanup-when-delete-backup: {{ .Values.defaultSettings.autoCleanupSnapshotWhenDeleteBackup }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSnapshotAfterOnDemandBackupCompleted) }} auto-cleanup-snapshot-after-on-demand-backup-completed: {{ .Values.defaultSettings.autoCleanupSnapshotAfterOnDemandBackupCompleted }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.rwxVolumeFastFailover) }} rwx-volume-fast-failover: {{ .Values.defaultSettings.rwxVolumeFastFailover}} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.offlineReplicaRebuilding) }} offline-replica-rebuilding: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.offlineReplicaRebuilding }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.dataEngineCPUMask) }} data-engine-cpu-mask: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.dataEngineCPUMask }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaRebuildingBandwidthLimit) }} replica-rebuilding-bandwidth-limit: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.replicaRebuildingBandwidthLimit }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaRebuildingBandwidthLimit) }} default-ublk-queue-depth: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.defaultUblkQueueDepth }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.replicaRebuildingBandwidthLimit) }} default-ublk-number-of-queue: {{ include "longhorn.multiTypeSetting" .Values.defaultSettings.defaultUblkNumberOfQueue }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.instanceManagerPodLivenessProbeTimeout) }} instance-manager-pod-liveness-probe-timeout: {{ .Values.defaultSettings.instanceManagerPodLivenessProbeTimeout | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.snapshotHeavyTaskConcurrentLimit) }} snapshot-heavy-task-concurrent-limit: {{ .Values.defaultSettings.snapshotHeavyTaskConcurrentLimit | quote }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.nodeDiskHealthMonitoring) }} node-disk-health-monitoring: {{ .Values.defaultSettings.nodeDiskHealthMonitoring }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultSettings.csiAllowedTopologyKeys) }} csi-allowed-topology-keys: {{ .Values.defaultSettings.csiAllowedTopologyKeys | quote }} {{- end }} ================================================ FILE: chart/templates/deployment-driver.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-driver-deployer namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: replicas: 1 selector: matchLabels: app: longhorn-driver-deployer template: metadata: labels: {{- include "longhorn.labels" . | nindent 8 }} app: longhorn-driver-deployer spec: initContainers: - name: wait-longhorn-manager image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - longhorn-manager - -d {{- if eq .Values.longhornDriver.log.format "json" }} - -j {{- end }} - deploy-driver - --manager-image - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" - --manager-url - http://longhorn-backend:9500/v1 env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName {{- if .Values.csi.kubeletRootDir }} - name: KUBELET_ROOT_DIR value: {{ .Values.csi.kubeletRootDir }} {{- end }} {{- if .Values.csi.podAntiAffinityPreset }} - name: CSI_POD_ANTI_AFFINITY_PRESET value: {{ .Values.csi.podAntiAffinityPreset }} {{- end }} {{- if and .Values.image.csi.attacher.repository .Values.image.csi.attacher.tag }} - name: CSI_ATTACHER_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.attacher.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.attacher.repository }}:{{ .Values.image.csi.attacher.tag }}" {{- end }} {{- if and .Values.image.csi.provisioner.repository .Values.image.csi.provisioner.tag }} - name: CSI_PROVISIONER_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.provisioner.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.provisioner.repository }}:{{ .Values.image.csi.provisioner.tag }}" {{- end }} {{- if and .Values.image.csi.nodeDriverRegistrar.repository .Values.image.csi.nodeDriverRegistrar.tag }} - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.nodeDriverRegistrar.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.nodeDriverRegistrar.repository }}:{{ .Values.image.csi.nodeDriverRegistrar.tag }}" {{- end }} {{- if and .Values.image.csi.resizer.repository .Values.image.csi.resizer.tag }} - name: CSI_RESIZER_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.resizer.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.resizer.repository }}:{{ .Values.image.csi.resizer.tag }}" {{- end }} {{- if and .Values.image.csi.snapshotter.repository .Values.image.csi.snapshotter.tag }} - name: CSI_SNAPSHOTTER_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.snapshotter.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.snapshotter.repository }}:{{ .Values.image.csi.snapshotter.tag }}" {{- end }} {{- if and .Values.image.csi.livenessProbe.repository .Values.image.csi.livenessProbe.tag }} - name: CSI_LIVENESS_PROBE_IMAGE value: "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.csi.livenessProbe.registry) }}{{ . }}/{{ end }}{{ .Values.image.csi.livenessProbe.repository }}:{{ .Values.image.csi.livenessProbe.tag }}" {{- end }} {{- if .Values.csi.attacherReplicaCount }} - name: CSI_ATTACHER_REPLICA_COUNT value: {{ .Values.csi.attacherReplicaCount | quote }} {{- end }} {{- if .Values.csi.provisionerReplicaCount }} - name: CSI_PROVISIONER_REPLICA_COUNT value: {{ .Values.csi.provisionerReplicaCount | quote }} {{- end }} {{- if .Values.csi.resizerReplicaCount }} - name: CSI_RESIZER_REPLICA_COUNT value: {{ .Values.csi.resizerReplicaCount | quote }} {{- end }} {{- if .Values.csi.snapshotterReplicaCount }} - name: CSI_SNAPSHOTTER_REPLICA_COUNT value: {{ .Values.csi.snapshotterReplicaCount | quote }} {{- end }} {{- if .Values.enableGoCoverDir }} - name: GOCOVERDIR value: /go-cover-dir/ volumeMounts: - name: go-cover-dir mountPath: /go-cover-dir/ {{- end }} {{- include "longhorn.timezoneEnv" . | nindent 10 }} {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.longhornDriver.priorityClass }} priorityClassName: {{ .Values.longhornDriver.priorityClass | quote }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornDriver.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornDriver.tolerations }} {{ default .Values.global.tolerations .Values.longhornDriver.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornDriver.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornDriver.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornDriver.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} serviceAccountName: longhorn-service-account securityContext: runAsUser: 0 {{- if .Values.enableGoCoverDir }} volumes: - name: go-cover-dir hostPath: path: /go-cover-dir/ type: DirectoryOrCreate {{- end }} ================================================ FILE: chart/templates/deployment-ui.yaml ================================================ {{- if .Values.openshift.enabled }} {{- if .Values.openshift.ui.route }} # https://github.com/openshift/oauth-proxy/blob/master/contrib/sidecar.yaml # Create a proxy service account and ensure it will use the route "proxy" # Create a secure connection to the proxy via a route apiVersion: route.openshift.io/v1 kind: Route metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ui name: {{ .Values.openshift.ui.route }} namespace: {{ include "release_namespace" . }} spec: to: kind: Service name: longhorn-ui tls: termination: reencrypt --- apiVersion: v1 kind: Service metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ui name: longhorn-ui namespace: {{ include "release_namespace" . }} annotations: service.alpha.openshift.io/serving-cert-secret-name: longhorn-ui-tls spec: ports: - name: longhorn-ui port: {{ .Values.openshift.ui.port | default 443 }} targetPort: {{ .Values.openshift.ui.proxy | default 8443 }} selector: app: longhorn-ui --- {{- end }} {{- end }} apiVersion: apps/v1 kind: Deployment metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ui name: longhorn-ui namespace: {{ include "release_namespace" . }} spec: replicas: {{ .Values.longhornUI.replicas }} selector: matchLabels: app: longhorn-ui template: metadata: labels: {{- include "longhorn.labels" . | nindent 8 }} app: longhorn-ui spec: serviceAccountName: longhorn-ui-service-account affinity: {{- toYaml .Values.longhornUI.affinity | nindent 8 }} containers: {{- if .Values.openshift.enabled }} {{- if .Values.openshift.ui.route }} - name: oauth-proxy {{- if .Values.image.openshift.oauthProxy.repository }} image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.openshift.oauthProxy.registry) }}{{ . }}/{{ end }}{{ .Values.image.openshift.oauthProxy.repository }}:{{ .Values.image.openshift.oauthProxy.tag }} {{- else }} image: "" {{- end }} imagePullPolicy: IfNotPresent ports: - containerPort: {{ .Values.openshift.ui.proxy | default 8443 }} name: public args: - --https-address=:{{ .Values.openshift.ui.proxy | default 8443 }} - --provider=openshift - --openshift-service-account=longhorn-ui-service-account - --upstream=http://localhost:8000 - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET - --openshift-sar={"namespace":"{{ include "release_namespace" . }}","group":"longhorn.io","resource":"setting","verb":"delete"} volumeMounts: - mountPath: /etc/tls/private name: longhorn-ui-tls {{- end }} {{- end }} - name: longhorn-ui image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.ui.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.ui.repository }}:{{ .Values.image.longhorn.ui.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: nginx-cache mountPath: /var/cache/nginx/ - name: nginx-config mountPath: /var/config/nginx/ - name: var-run mountPath: /var/run/ ports: - containerPort: 8000 name: http env: - name: LONGHORN_MANAGER_IP value: "http://longhorn-backend:9500" - name: LONGHORN_UI_PORT value: "8000" {{- include "longhorn.timezoneEnv" . | nindent 10 }} volumes: {{- if .Values.openshift.enabled }} {{- if .Values.openshift.ui.route }} - name: longhorn-ui-tls secret: secretName: longhorn-ui-tls {{- end }} {{- end }} - emptyDir: {} name: nginx-cache - emptyDir: {} name: nginx-config - emptyDir: {} name: var-run {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.longhornUI.priorityClass }} priorityClassName: {{ .Values.longhornUI.priorityClass | quote }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornUI.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornUI.tolerations }} {{ default .Values.global.tolerations .Values.longhornUI.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornUI.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornUI.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornUI.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} --- kind: Service apiVersion: v1 metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ui {{- if eq .Values.service.ui.type "Rancher-Proxy" }} kubernetes.io/cluster-service: "true" {{- end }} {{- with .Values.service.ui.labels }} {{- toYaml . | nindent 4 }} {{- end }} name: longhorn-frontend namespace: {{ include "release_namespace" . }} {{- with .Values.service.ui.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: {{- if eq .Values.service.ui.type "Rancher-Proxy" }} type: ClusterIP {{- else }} type: {{ .Values.service.ui.type }} {{- end }} {{- if and .Values.service.ui.loadBalancerIP (eq .Values.service.ui.type "LoadBalancer") }} loadBalancerIP: {{ .Values.service.ui.loadBalancerIP }} {{- end }} {{- if and (eq .Values.service.ui.type "LoadBalancer") .Values.service.ui.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{- toYaml .Values.service.ui.loadBalancerSourceRanges | nindent 4 }} {{- end }} {{- if and (eq .Values.service.ui.type "LoadBalancer") .Values.service.ui.loadBalancerClass }} loadBalancerClass: {{ .Values.service.ui.loadBalancerClass }} {{- end }} selector: app: longhorn-ui ports: - name: http port: 80 targetPort: http {{- if .Values.service.ui.nodePort }} nodePort: {{ .Values.service.ui.nodePort }} {{- else }} nodePort: null {{- end }} ================================================ FILE: chart/templates/extra-objects.yaml ================================================ {{- range .Values.extraObjects }} --- {{- tpl (toYaml . ) $ }} {{- end }} ================================================ FILE: chart/templates/httproute.yaml ================================================ {{- if .Values.httproute.enabled -}} apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: longhorn-httproute namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-httproute {{- with .Values.httproute.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: {{- with .Values.httproute.parentRefs }} parentRefs: {{- range . }} - group: {{ .group | default "gateway.networking.k8s.io" }} kind: {{ .kind | default "Gateway" }} name: {{ .name }} {{- with .namespace }} namespace: {{ . }} {{- end }} {{- with .sectionName }} sectionName: {{ . }} {{- end }} {{- end }} {{- end }} {{- with .Values.httproute.hostnames }} hostnames: {{- range . }} - {{ . | quote }} {{- end }} {{- end }} rules: - matches: - path: type: {{ .Values.httproute.pathType | default "PathPrefix" }} value: {{ .Values.httproute.path | default "/" }} backendRefs: - name: longhorn-frontend port: 80 {{- end }} ================================================ FILE: chart/templates/ingress.yaml ================================================ {{- if .Values.ingress.enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: longhorn-ingress namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ingress annotations: {{- if .Values.ingress.secureBackends }} ingress.kubernetes.io/secure-backends: "true" {{- end }} {{- range $key, $value := .Values.ingress.annotations }} {{ $key }}: {{ $value | quote }} {{- end }} spec: {{- if .Values.ingress.ingressClassName }} ingressClassName: {{ .Values.ingress.ingressClassName }} {{- end }} rules: - host: {{ .Values.ingress.host }} http: paths: - path: {{ default "" .Values.ingress.path }} pathType: {{ default "ImplementationSpecific" .Values.ingress.pathType }} backend: service: name: longhorn-frontend port: number: 80 {{- range .Values.ingress.extraHosts }} - host: {{ . }} http: paths: - path: {{ default "" $.Values.ingress.path }} pathType: {{ default "ImplementationSpecific" $.Values.ingress.pathType }} backend: service: name: longhorn-frontend port: number: 80 {{- end }} {{- if .Values.ingress.tls }} tls: - hosts: - {{ .Values.ingress.host }} {{- range .Values.ingress.extraHosts }} - {{ . }} {{- end }} secretName: {{ .Values.ingress.tlsSecret }} {{- end }} {{- end }} ================================================ FILE: chart/templates/network-policies/backing-image-data-source-network-policy.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backing-image-data-source namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: longhorn.io/component: backing-image-data-source policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-manager - podSelector: matchLabels: longhorn.io/component: backing-image-data-source {{- end }} ================================================ FILE: chart/templates/network-policies/backing-image-manager-network-policy.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backing-image-manager namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: longhorn.io/component: backing-image-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-manager - podSelector: matchLabels: longhorn.io/component: backing-image-data-source {{- end }} ================================================ FILE: chart/templates/network-policies/instance-manager-networking.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: instance-manager namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: longhorn.io/component: instance-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-manager - podSelector: matchLabels: longhorn.io/component: backing-image-data-source {{- end }} ================================================ FILE: chart/templates/network-policies/manager-network-policy.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-manager namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: app: longhorn-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: app: longhorn-ui - podSelector: matchLabels: app: longhorn-csi-plugin - podSelector: matchLabels: longhorn.io/managed-by: longhorn-manager matchExpressions: - { key: recurring-job.longhorn.io, operator: Exists } - podSelector: matchExpressions: - { key: longhorn.io/job-task, operator: Exists } - podSelector: matchLabels: app: longhorn-driver-deployer {{- end }} ================================================ FILE: chart/templates/network-policies/recovery-backend-network-policy.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-recovery-backend namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: longhorn.io/recovery-backend: longhorn-recovery-backend policyTypes: - Ingress ingress: - ports: - protocol: TCP port: 9503 {{- end }} ================================================ FILE: chart/templates/network-policies/ui-frontend-network-policy.yaml ================================================ {{- if and .Values.networkPolicies.enabled .Values.ingress.enabled (not (eq .Values.networkPolicies.type "")) }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-ui-frontend namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: app: longhorn-ui policyTypes: - Ingress ingress: - from: {{- if eq .Values.networkPolicies.type "rke1"}} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: ingress-nginx podSelector: matchLabels: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx {{- else if eq .Values.networkPolicies.type "rke2" }} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system podSelector: matchLabels: app.kubernetes.io/component: controller app.kubernetes.io/instance: rke2-ingress-nginx app.kubernetes.io/name: rke2-ingress-nginx {{- else if eq .Values.networkPolicies.type "k3s" }} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system podSelector: matchLabels: app.kubernetes.io/name: traefik ports: - port: 8000 protocol: TCP - port: 80 protocol: TCP {{- end }} {{- end }} ================================================ FILE: chart/templates/network-policies/webhook-network-policy.yaml ================================================ {{- if .Values.networkPolicies.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-admission-webhook namespace: {{ include "release_namespace" . }} spec: podSelector: matchLabels: longhorn.io/admission-webhook: longhorn-admission-webhook policyTypes: - Ingress ingress: - ports: - protocol: TCP port: 9502 {{- end }} ================================================ FILE: chart/templates/postupgrade-job.yaml ================================================ apiVersion: batch/v1 kind: Job metadata: annotations: "helm.sh/hook": post-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation name: longhorn-post-upgrade namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: activeDeadlineSeconds: 900 backoffLimit: 1 template: metadata: name: longhorn-post-upgrade labels: {{- include "longhorn.labels" . | nindent 8 }} spec: containers: - name: longhorn-post-upgrade image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - longhorn-manager - post-upgrade env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {{- include "longhorn.timezoneEnv" . | nindent 8 }} restartPolicy: OnFailure {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.longhornManager.priorityClass }} priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} {{- end }} serviceAccountName: longhorn-service-account {{- if or .Values.global.tolerations .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornManager.tolerations }} {{ default .Values.global.tolerations .Values.longhornManager.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornManager.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} ================================================ FILE: chart/templates/preupgrade-job.yaml ================================================ {{- if and .Values.preUpgradeChecker.jobEnabled .Values.preUpgradeChecker.upgradeVersionCheck}} apiVersion: batch/v1 kind: Job metadata: annotations: "helm.sh/hook": pre-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed name: longhorn-pre-upgrade namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: activeDeadlineSeconds: 900 backoffLimit: 1 template: metadata: name: longhorn-pre-upgrade labels: {{- include "longhorn.labels" . | nindent 8 }} spec: containers: - name: longhorn-pre-upgrade image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: privileged: true command: - longhorn-manager - pre-upgrade volumeMounts: - name: proc mountPath: /host/proc/ env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {{- include "longhorn.timezoneEnv" . | nindent 8 }} volumes: - name: proc hostPath: path: /proc/ restartPolicy: OnFailure {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} serviceAccountName: longhorn-service-account {{- if or .Values.global.tolerations .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornManager.tolerations }} {{ default .Values.global.tolerations .Values.longhornManager.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornManager.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} {{- end }} ================================================ FILE: chart/templates/priorityclass.yaml ================================================ apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: "longhorn-critical" labels: {{- include "longhorn.labels" . | nindent 4 }} description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" globalDefault: false preemptionPolicy: PreemptLowerPriority value: 1000000000 ================================================ FILE: chart/templates/psp.yaml ================================================ {{- if .Values.enablePSP }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: longhorn-psp labels: {{- include "longhorn.labels" . | nindent 4 }} spec: privileged: true allowPrivilegeEscalation: true requiredDropCapabilities: - NET_RAW allowedCapabilities: - SYS_ADMIN hostNetwork: false hostIPC: false hostPID: true runAsUser: rule: RunAsAny seLinux: rule: RunAsAny fsGroup: rule: RunAsAny supplementalGroups: rule: RunAsAny volumes: - configMap - downwardAPI - emptyDir - secret - projected - hostPath --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: longhorn-psp-role labels: {{- include "longhorn.labels" . | nindent 4 }} namespace: {{ include "release_namespace" . }} rules: - apiGroups: - policy resources: - podsecuritypolicies verbs: - use resourceNames: - longhorn-psp --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: longhorn-psp-binding labels: {{- include "longhorn.labels" . | nindent 4 }} namespace: {{ include "release_namespace" . }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: longhorn-psp-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: {{ include "release_namespace" . }} - kind: ServiceAccount name: default namespace: {{ include "release_namespace" . }} {{- end }} ================================================ FILE: chart/templates/registry-secret.yaml ================================================ {{- if .Values.privateRegistry.createSecret }} {{- if .Values.privateRegistry.registrySecret }} {{- if not (kindIs "string" .Values.privateRegistry.registrySecret) }} {{- fail "The privateRegistry.registrySecret value must be a string" }} {{- end }} apiVersion: v1 kind: Secret metadata: name: {{ .Values.privateRegistry.registrySecret }} namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: {{ template "secret" . }} {{- end }} {{- end }} ================================================ FILE: chart/templates/role.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ include "longhorn.name" . }} namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["pods", "pods/log", "events", "secrets", "services", "endpoints", "configmaps", "serviceaccounts", "persistentvolumeclaims", "persistentvolumeclaims/status"] verbs: ["*"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "statefulsets", "replicasets"] verbs: ["*"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["*"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["*"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["*"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["*"] ================================================ FILE: chart/templates/rolebinding.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ include "longhorn.name" . }} namespace: {{ include "release_namespace" . }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: {{ include "longhorn.name" . }} subjects: - kind: ServiceAccount name: longhorn-service-account namespace: {{ include "release_namespace" . }} ================================================ FILE: chart/templates/serviceaccount.yaml ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-service-account namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} --- apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-ui-service-account namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} {{- if .Values.openshift.enabled }} {{- if .Values.openshift.ui.route }} {{- if not .Values.serviceAccount.annotations }} annotations: {{- end }} serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"longhorn-ui"}}' {{- end }} {{- end }} --- apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-support-bundle namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} ================================================ FILE: chart/templates/servicemonitor.yaml ================================================ {{- if .Values.metrics.serviceMonitor.enabled -}} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: longhorn-prometheus-servicemonitor namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} name: longhorn-prometheus-servicemonitor {{- with .Values.metrics.serviceMonitor.additionalLabels }} {{- toYaml . | nindent 4 }} {{- end }} {{- with .Values.metrics.serviceMonitor.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: selector: matchLabels: app: longhorn-manager namespaceSelector: matchNames: - {{ include "release_namespace" . }} endpoints: - port: manager {{- with .Values.metrics.serviceMonitor.interval }} interval: {{ . }} {{- end }} {{- with .Values.metrics.serviceMonitor.scrapeTimeout }} scrapeTimeout: {{ . }} {{- end }} {{- with .Values.metrics.serviceMonitor.relabelings }} relabelings: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.metrics.serviceMonitor.metricRelabelings }} metricRelabelings: {{- toYaml . | nindent 8 }} {{- end }} {{- end }} ================================================ FILE: chart/templates/services.yaml ================================================ apiVersion: v1 kind: Service metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-admission-webhook name: longhorn-admission-webhook namespace: {{ include "release_namespace" . }} spec: type: ClusterIP selector: longhorn.io/admission-webhook: longhorn-admission-webhook ports: - name: admission-webhook port: 9502 targetPort: admission-wh --- apiVersion: v1 kind: Service metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-recovery-backend name: longhorn-recovery-backend namespace: {{ include "release_namespace" . }} spec: type: ClusterIP selector: longhorn.io/recovery-backend: longhorn-recovery-backend ports: - name: recovery-backend port: 9503 targetPort: recov-backend ================================================ FILE: chart/templates/storageclass.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: name: longhorn-storageclass namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} data: storageclass.yaml: | kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn annotations: storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }} provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: "{{ .Values.persistence.reclaimPolicy }}" volumeBindingMode: {{ .Values.persistence.volumeBindingMode | default "Immediate" }} parameters: numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}" staleReplicaTimeout: "30" fromBackup: "" {{- if .Values.persistence.defaultFsType }} fsType: "{{ .Values.persistence.defaultFsType }}" {{- end }} {{- if .Values.persistence.defaultMkfsParams }} mkfsParams: "{{ .Values.persistence.defaultMkfsParams }}" {{- end }} {{- if .Values.persistence.migratable }} migratable: "{{ .Values.persistence.migratable }}" {{- end }} {{- if .Values.persistence.nfsOptions }} nfsOptions: "{{ .Values.persistence.nfsOptions }}" {{- end }} {{- if .Values.persistence.backingImage.enable }} backingImage: {{ .Values.persistence.backingImage.name }} backingImageDataSourceType: {{ .Values.persistence.backingImage.dataSourceType }} backingImageDataSourceParameters: {{ .Values.persistence.backingImage.dataSourceParameters }} backingImageChecksum: {{ .Values.persistence.backingImage.expectedChecksum }} {{- end }} {{- if .Values.persistence.recurringJobSelector.enable }} recurringJobSelector: '{{ .Values.persistence.recurringJobSelector.jobList }}' {{- end }} dataLocality: {{ .Values.persistence.defaultDataLocality | quote }} {{- if .Values.persistence.defaultDiskSelector.enable }} diskSelector: "{{ .Values.persistence.defaultDiskSelector.selector }}" {{- end }} {{- if .Values.persistence.defaultNodeSelector.enable }} nodeSelector: "{{ .Values.persistence.defaultNodeSelector.selector }}" {{- end }} {{- if .Values.persistence.unmapMarkSnapChainRemoved }} unmapMarkSnapChainRemoved: "{{ .Values.persistence.unmapMarkSnapChainRemoved }}" {{- end }} {{- if .Values.persistence.disableRevisionCounter }} disableRevisionCounter: "{{ .Values.persistence.disableRevisionCounter }}" dataEngine: "{{ .Values.persistence.dataEngine }}" {{- end }} {{- if .Values.persistence.backupTargetName }} backupTargetName: "{{ .Values.persistence.backupTargetName }}" {{- end }} ================================================ FILE: chart/templates/tls-secrets.yaml ================================================ {{- if .Values.ingress.enabled }} {{- range .Values.ingress.secrets }} apiVersion: v1 kind: Secret metadata: name: {{ .name }} namespace: {{ include "release_namespace" $ }} labels: {{- include "longhorn.labels" $ | nindent 4 }} app: longhorn type: kubernetes.io/tls data: tls.crt: {{ .certificate | b64enc }} tls.key: {{ .key | b64enc }} --- {{- end }} {{- end }} ================================================ FILE: chart/templates/uninstall-job.yaml ================================================ apiVersion: batch/v1 kind: Job metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded name: longhorn-uninstall namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: activeDeadlineSeconds: 900 backoffLimit: 1 template: metadata: name: longhorn-uninstall labels: {{- include "longhorn.labels" . | nindent 8 }} spec: containers: - name: longhorn-uninstall image: {{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.manager.registry) }}{{ . }}/{{ end }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - longhorn-manager - uninstall - --force env: - name: LONGHORN_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {{- include "longhorn.timezoneEnv" . | nindent 8 }} restartPolicy: Never {{- with (coalesce .Values.global.imagePullSecrets .Values.privateRegistry.registrySecret) }} imagePullSecrets: {{- $imagePullSecrets := list }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- range . }} {{- if kindIs "string" . }} {{- $imagePullSecrets = append $imagePullSecrets (dict "name" .) }} {{- else }} {{- $imagePullSecrets = append $imagePullSecrets . }} {{- end }} {{- end }} {{- end }} {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.longhornManager.priorityClass }} priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} {{- end }} serviceAccountName: longhorn-service-account {{- if or .Values.global.tolerations .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} tolerations: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} {{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} {{- end }} {{- if or .Values.global.tolerations .Values.longhornManager.tolerations }} {{ default .Values.global.tolerations .Values.longhornManager.tolerations | toYaml | indent 6 }} {{- end }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} nodeSelector: {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} {{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} {{- end }} {{- if or .Values.global.nodeSelector .Values.longhornManager.nodeSelector }} {{ default .Values.global.nodeSelector .Values.longhornManager.nodeSelector | toYaml | indent 8 }} {{- end }} {{- end }} ================================================ FILE: chart/templates/validate-psp-install.yaml ================================================ #{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} #{{- if .Values.enablePSP }} #{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} #{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} #{{- end }} #{{- end }} #{{- end }} ================================================ FILE: chart/values.yaml ================================================ # Default values for longhorn. # This is a YAML-formatted file. # Declare variables to be passed into your templates. global: # -- Global override for container image registry. imageRegistry: "docker.io" # -- Global override for image pull secrets for container registry. imagePullSecrets: [] # -- Set container timezone (TZ env) for all Longhorn workloads. Leave empty to use container default. timezone: "" # -- Toleration for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer. tolerations: [] # -- Node selector for nodes allowed to run user-deployed components such as Longhorn Manager, Longhorn UI, and Longhorn Driver Deployer. nodeSelector: {} cattle: # -- Default system registry. systemDefaultRegistry: "" windowsCluster: # -- Setting that allows Longhorn to run on a Rancher Windows cluster. enabled: false # -- Toleration for Linux nodes that can run user-deployed Longhorn components. tolerations: - key: "cattle.io/os" value: "linux" effect: "NoSchedule" operator: "Equal" # -- Node selector for Linux nodes that can run user-deployed Longhorn components. nodeSelector: kubernetes.io/os: "linux" defaultSetting: # -- Toleration for system-managed Longhorn components. taintToleration: cattle.io/os=linux:NoSchedule # -- Node selector for system-managed Longhorn components. systemManagedComponentsNodeSelector: kubernetes.io/os:linux networkPolicies: # -- Setting that allows you to enable network policies that control access to Longhorn pods. enabled: false # -- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") type: "k3s" image: longhorn: engine: # -- Registry for the Longhorn Engine image. registry: "" # -- Repository for the Longhorn Engine image. repository: longhornio/longhorn-engine # -- Tag for the Longhorn Engine image. tag: master-head manager: # -- Registry for the Longhorn Manager image. registry: "" # -- Repository for the Longhorn Manager image. repository: longhornio/longhorn-manager # -- Tag for the Longhorn Manager image. tag: master-head ui: # -- Registry for the Longhorn UI image. registry: "" # -- Repository for the Longhorn UI image. repository: longhornio/longhorn-ui # -- Tag for the Longhorn UI image. tag: master-head instanceManager: # -- Registry for the Longhorn Instance Manager image. registry: "" # -- Repository for the Longhorn Instance Manager image. repository: longhornio/longhorn-instance-manager # -- Tag for the Longhorn Instance Manager image. tag: master-head shareManager: # -- Registry for the Longhorn Share Manager image. registry: "" # -- Repository for the Longhorn Share Manager image. repository: longhornio/longhorn-share-manager # -- Tag for the Longhorn Share Manager image. tag: master-head backingImageManager: # -- Registry for the Backing Image Manager image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. repository: longhornio/backing-image-manager # -- Tag for the Backing Image Manager image. When unspecified, Longhorn uses the default value. tag: master-head supportBundleKit: # -- Registry for the Longhorn Support Bundle Manager image. registry: "" # -- Repository for the Longhorn Support Bundle Manager image. repository: longhornio/support-bundle-kit # -- Tag for the Longhorn Support Bundle Manager image. tag: v0.0.79 csi: attacher: # -- Registry for the CSI attacher image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. repository: longhornio/csi-attacher # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. tag: v4.10.0-20251226 provisioner: # -- Registry for the CSI Provisioner image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. repository: longhornio/csi-provisioner # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. tag: v5.3.0-20251226 nodeDriverRegistrar: # -- Registry for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. repository: longhornio/csi-node-driver-registrar # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. tag: v2.15.0-20251226 resizer: # -- Registry for the CSI Resizer image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. repository: longhornio/csi-resizer # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. tag: v2.0.0-20251226 snapshotter: # -- Registry for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. repository: longhornio/csi-snapshotter # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. tag: v8.4.0-20251226 livenessProbe: # -- Registry for the CSI liveness probe image. When unspecified, Longhorn uses the default value. registry: "" # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. repository: longhornio/livenessprobe # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. tag: v2.17.0-20251226 openshift: oauthProxy: # -- Registry for the OAuth Proxy image. Specify the upstream image (for example, "quay.io/openshift/origin-oauth-proxy"). This setting applies only to OpenShift users. registry: "" # -- Repository for the OAuth Proxy image. Specify the upstream image (for example, "quay.io/openshift/origin-oauth-proxy"). This setting applies only to OpenShift users. repository: "" # -- Tag for the OAuth Proxy image. Specify OCP/OKD version 4.1 or later (including version 4.18, which is available at quay.io/openshift/origin-oauth-proxy:4.18). This setting applies only to OpenShift users. tag: "" # -- Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI. pullPolicy: IfNotPresent service: ui: # -- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") type: ClusterIP # -- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. nodePort: null # -- Class of a load balancer implementation loadBalancerClass: "" # -- Annotation for the Longhorn UI service. annotations: {} ## If you want to set annotations for the Longhorn UI service, delete the `{}` in the line above ## and uncomment this example block # annotation-key1: "annotation-value1" # annotation-key2: "annotation-value2" labels: {} ## If you want to set additional labels for the Longhorn UI service, delete the `{}` in the line above ## and uncomment this example block # label-key1: "label-value1" # label-key2: "label-value2" manager: # -- Service type for Longhorn Manager. type: ClusterIP # -- NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767. nodePort: "" persistence: # -- Setting that allows you to specify the default Longhorn StorageClass. defaultClass: true # -- Filesystem type of the default Longhorn StorageClass. defaultFsType: ext4 # -- mkfs parameters of the default Longhorn StorageClass. defaultMkfsParams: "" # -- Replica count of the default Longhorn StorageClass. defaultClassReplicaCount: 3 # -- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") defaultDataLocality: disabled # -- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") reclaimPolicy: Delete # -- VolumeBindingMode controls when volume binding and dynamic provisioning should occur. (Options: "Immediate", "WaitForFirstConsumer") (Defaults to "Immediate") volumeBindingMode: "Immediate" # -- Setting that allows you to enable live migration of a Longhorn volume from one node to another. migratable: false # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the volume-head-xxx.img file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. disableRevisionCounter: "true" # -- Set NFS mount options for Longhorn StorageClass for RWX volumes nfsOptions: "" recurringJobSelector: # -- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. enable: false # -- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) jobList: [] backingImage: # -- Setting that allows you to use a backing image in a Longhorn StorageClass. enable: false # -- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. name: ~ # -- Data source type of a backing image used in a Longhorn StorageClass. # If the backing image exists in the cluster, Longhorn uses this setting to verify the image. # If the backing image does not exist, Longhorn creates one using the specified data source type. dataSourceType: ~ # -- Data source parameters of a backing image used in a Longhorn StorageClass. # You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) dataSourceParameters: ~ # -- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. expectedChecksum: ~ defaultDiskSelector: # -- Setting that allows you to enable the disk selector for the default Longhorn StorageClass. enable: false # -- Disk selector for the default Longhorn StorageClass. Longhorn uses only disks with the specified tags for storing volume data. (Examples: "nvme,sata") selector: "" defaultNodeSelector: # -- Setting that allows you to enable the node selector for the default Longhorn StorageClass. enable: false # -- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") selector: "" # -- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") unmapMarkSnapChainRemoved: ignored # -- Setting that allows you to specify the data engine version for the default Longhorn StorageClass. (Options: "v1", "v2") dataEngine: v1 # -- Setting that allows you to specify the backup target for the default Longhorn StorageClass. backupTargetName: default preUpgradeChecker: # -- Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions. jobEnabled: true # -- Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled. upgradeVersionCheck: true csi: # -- kubelet root directory. When unspecified, Longhorn uses the default value. kubeletRootDir: ~ # -- Configures Pod anti-affinity to prevent multiple instances on the same node. Use soft (tries to separate) or hard (must separate). When unspecified, Longhorn uses the default value ("soft"). podAntiAffinityPreset: ~ # -- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). attacherReplicaCount: ~ # -- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). provisionerReplicaCount: ~ # -- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). resizerReplicaCount: ~ # -- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). snapshotterReplicaCount: ~ defaultSettings: # -- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. allowRecurringJobWhileVolumeDetached: ~ # -- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. createDefaultDiskLabeledNodes: ~ # -- Default path to use for storing data on a host. An absolute directory path indicates a filesystem-type disk used by the V1 Data Engine, while a path to a block device indicates a block-type disk used by the V2 Data Engine. The default value is "/var/lib/longhorn/". defaultDataPath: ~ # -- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. defaultDataLocality: ~ # -- Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default. replicaSoftAntiAffinity: ~ # -- Setting that automatically rebalances replicas when an available node is discovered. replicaAutoBalance: ~ # -- Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100". storageOverProvisioningPercentage: ~ # -- Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25". storageMinimalAvailablePercentage: ~ # -- Percentage of disk space that is not allocated to the default disk on each new Longhorn node. storageReservedPercentageForDefaultDisk: ~ # -- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default upgradeChecker: ~ # -- The Upgrade Responder sends a notification whenever a new Longhorn version that you can upgrade to becomes available. The default value is https://longhorn-upgrade-responder.rancher.io/v1/checkupgrade. upgradeResponderURL: ~ # -- The external URL used to access the Longhorn Manager API. When set, this URL is returned in API responses (the actions and links fields) instead of the internal pod IP. This is useful when accessing the API through Ingress or Gateway API HTTPRoute. Format: scheme://host[:port] (for example, https://longhorn.example.com or https://longhorn.example.com:8443). Leave it empty to use the default behavior. managerUrl: ~ # -- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "{"v1":"3","v2":"3"}". defaultReplicaCount: ~ # -- Default name of Longhorn static StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. "storageClassName" needs to be an existing StorageClass. The default value is "longhorn-static". defaultLonghornStaticStorageClass: ~ # -- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. failedBackupTTL: ~ # -- Number of minutes that Longhorn allows for the backup execution. The default value is "1". backupExecutionTimeout: ~ # -- Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration. restoreVolumeRecurringJobs: ~ # -- Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained. recurringSuccessfulJobsHistoryLimit: ~ # -- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. recurringFailedJobsHistoryLimit: ~ # -- Maximum number of snapshots or backups to be retained. recurringJobMaxRetention: ~ # -- Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles. supportBundleFailedHistoryLimit: ~ # -- Taint or toleration for system-managed Longhorn components. # Specify values using a semicolon-separated list in `kubectl taint` syntax (Example: key1=value1:effect; key2=value2:effect). taintToleration: ~ # -- Node selector for system-managed Longhorn components. systemManagedComponentsNodeSelector: ~ # -- Resource limits for system-managed CSI components. # This setting allows you to configure CPU and memory requests/limits for CSI attacher, provisioner, resizer, snapshotter, and plugin components. # Supported components: csi-attacher, csi-provisioner, csi-resizer, csi-snapshotter, longhorn-csi-plugin, node-driver-registrar, longhorn-liveness-probe. # Notice that changing resource limits will cause CSI components to restart, which may temporarily affect volume provisioning and attach/detach operations until the components are ready. The value should be a JSON object with component names as keys and ResourceRequirements as values. systemManagedCSIComponentsResourceLimits: ~ # -- PriorityClass for system-managed Longhorn components. # This setting can help prevent Longhorn components from being evicted under Node Pressure. # Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. priorityClass: &defaultPriorityClassNameRef "longhorn-critical" # -- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. autoSalvage: ~ # -- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. autoDeletePodWhenVolumeDetachedUnexpectedly: ~ # -- Blacklist of controller api/kind values for the setting Automatically Delete Workload Pod when the Volume Is Detached Unexpectedly. If a workload pod is managed by a controller whose api/kind is listed in this blacklist, Longhorn will not automatically delete the pod when its volume is unexpectedly detached. Multiple controller api/kind entries can be specified, separated by semicolons. For example: `apps/StatefulSet;apps/DaemonSet`. Note that the controller api/kind is case sensitive and must exactly match the api/kind in the workload pod's owner reference. blacklistForAutoDeletePodWhenVolumeDetachedUnexpectedly: ~ # -- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. disableSchedulingOnCordonedNode: ~ # -- Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=" in the Kubernetes node object. replicaZoneSoftAntiAffinity: ~ # -- Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default. replicaDiskSoftAntiAffinity: ~ # -- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. nodeDownPodDeletionPolicy: ~ # -- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. nodeDrainPolicy: ~ # -- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. detachManuallyAttachedVolumesWhenCordoned: ~ # -- Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume. replicaReplenishmentWaitInterval: ~ # -- Maximum number of replicas that can be concurrently rebuilt on each node. concurrentReplicaRebuildPerNodeLimit: ~ # -- Maximum number of file synchronization operations that can run concurrently during a single replica rebuild. Right now, it's for v1 data engine only. rebuildConcurrentSyncLimit: ~ # -- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. concurrentVolumeBackupRestorePerNodeLimit: ~ # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. disableRevisionCounter: '{"v1":"true"}' # -- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. systemManagedPodsImagePullPolicy: ~ # -- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. allowVolumeCreationWithDegradedAvailability: ~ # -- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. autoCleanupSystemGeneratedSnapshot: ~ # -- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. autoCleanupRecurringJobBackupSnapshot: ~ # -- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. concurrentAutomaticEngineUpgradePerNodeLimit: ~ # -- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. backingImageCleanupWaitInterval: ~ # -- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". backingImageRecoveryWaitInterval: ~ # -- Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod. The default value is {"v1":"12","v2":"12"}. guaranteedInstanceManagerCPU: ~ # -- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. kubernetesClusterAutoscalerEnabled: ~ # -- Enables Longhorn to automatically delete orphaned resources and their associated data or processes (e.g., stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. # You need to specify the resource types to be deleted using a semicolon-separated list (e.g., `replica-data;instance`). Available items are: `replica-data`, `instance`. orphanResourceAutoDeletion: ~ # -- Specifies the wait time, in seconds, before Longhorn automatically deletes an orphaned Custom Resource (CR) and its associated resources. # Note that if a user manually deletes an orphaned CR, the deletion occurs immediately and does not respect this grace period. orphanResourceAutoDeletionGracePeriod: ~ # -- Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network. storageNetwork: ~ # -- Specifies a dedicated network for mounting RWX (ReadWriteMany) volumes. Leave this blank to use the default Kubernetes cluster network. **Caution**: This setting should change after all RWX volumes are detached because some Longhorn component pods must be recreated to apply the setting. You cannot modify this setting while RWX volumes are still attached. endpointNetworkForRWXVolume: ~ # -- Flag that prevents accidental uninstallation of Longhorn. deletingConfirmationFlag: ~ # -- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". engineReplicaTimeout: ~ # -- Setting that allows you to enable and disable snapshot hashing and data integrity checks. snapshotDataIntegrity: ~ # -- Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance. snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~ # -- Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format. snapshotDataIntegrityCronjob: ~ # -- Setting that controls how many snapshot heavy task operations (such as purge and clone) can run concurrently per node. This is a best-effort mechanism: due to the distributed nature of the system, temporary oversubscription may occur. The limiter reduces worst-case overload but does not guarantee perfect enforcement. snapshotHeavyTaskConcurrentLimit: ~ # -- Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files. removeSnapshotsDuringFilesystemTrim: ~ # -- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". fastReplicaRebuildEnabled: ~ # -- Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed. replicaFileSyncHttpClientTimeout: ~ # -- Number of seconds that Longhorn allows for the completion of replica rebuilding and snapshot cloning operations. longGRPCTimeOut: ~ # -- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") logLevel: ~ # -- Specifies the directory on the host where Longhorn stores log files for the instance manager pod. Currently, it is only used for instance manager pods in the v2 data engine. logPath: ~ # -- Setting that allows you to specify a backup compression method. backupCompressionMethod: ~ # -- Maximum number of worker threads that can concurrently run for each backup. backupConcurrentLimit: ~ # -- Specifies the default backup block size, in MiB, used when creating a new volume. Supported values are 2 or 16. defaultBackupBlockSize: ~ # -- Maximum number of worker threads that can concurrently run for each restore operation. restoreConcurrentLimit: ~ # -- Setting that allows you to enable the V1 Data Engine. v1DataEngine: ~ # -- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is an experimental feature and should not be used in production environments. v2DataEngine: ~ # -- Applies only to the V2 Data Engine. Enables hugepages for the Storage Performance Development Kit (SPDK) target daemon. If disabled, legacy memory is used. Allocation size is set via the Data Engine Memory Size setting. dataEngineHugepageEnabled: ~ # -- Applies only to the V2 Data Engine. Specifies the hugepage size, in MiB, for the Storage Performance Development Kit (SPDK) target daemon. The default value is "{"v2":"2048"}" dataEngineMemorySize: ~ # -- Applies only to the V2 Data Engine. Specifies the CPU cores on which the Storage Performance Development Kit (SPDK) target daemon runs. The daemon is deployed in each Instance Manager pod. Ensure that the number of assigned cores does not exceed the guaranteed Instance Manager CPUs for the V2 Data Engine. The default value is "{"v2":"0x1"}". dataEngineCPUMask: ~ # -- This setting specifies the default write bandwidth limit (in megabytes per second) for volume replica rebuilding when using the v2 data engine (SPDK). If this value is set to 0, there will be no write bandwidth limitation. Individual volumes can override this setting by specifying their own rebuilding bandwidth limit. replicaRebuildingBandwidthLimit: ~ # -- This setting specifies the default depth of each queue for Ublk frontend. This setting applies to volumes using the V2 Data Engine with Ublk front end. Individual volumes can override this setting by specifying their own Ublk queue depth. defaultUblkQueueDepth: ~ # -- This setting specifies the default the number of queues for ublk frontend. This setting applies to volumes using the V2 Data Engine with Ublk front end. Individual volumes can override this setting by specifying their own number of queues for ublk. defaultUblkNumberOfQueue: ~ # -- In seconds. The setting specifies the timeout for the instance manager pod liveness probe. The default value is 10 seconds. instanceManagerPodLivenessProbeTimeout: ~ # -- Setting that allows scheduling of empty node selector volumes to any node. allowEmptyNodeSelectorVolume: ~ # -- Setting that allows scheduling of empty disk selector volumes to any disk. allowEmptyDiskSelectorVolume: ~ # -- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. allowCollectingLonghornUsageMetrics: ~ # -- Setting that temporarily prevents all attempts to purge volume snapshots. disableSnapshotPurge: ~ # -- Maximum snapshot count for a volume. The value should be between 2 to 250 snapshotMaxCount: ~ # -- Applies only to the V2 Data Engine. Specifies the log level for the Storage Performance Development Kit (SPDK) target daemon. Supported values are: Error, Warning, Notice, Info, and Debug. The default is Notice. dataEngineLogLevel: ~ # -- Applies only to the V2 Data Engine. Specifies the log flags for the Storage Performance Development Kit (SPDK) target daemon. dataEngineLogFlags: ~ # -- Setting that freezes the filesystem on the root partition before a snapshot is created. freezeFilesystemForSnapshot: ~ # -- Setting that automatically cleans up the snapshot when the backup is deleted. autoCleanupSnapshotWhenDeleteBackup: ~ # -- Setting that automatically cleans up the snapshot after the on-demand backup is completed. autoCleanupSnapshotAfterOnDemandBackupCompleted: ~ # -- Setting that allows Longhorn to detect node failure and immediately migrate affected RWX volumes. rwxVolumeFastFailover: ~ # -- Enables automatic rebuilding of degraded replicas while the volume is detached. This setting only takes effect if the individual volume setting is set to `ignored` or `enabled`. offlineReplicaRebuilding: ~ # -- Controls whether Longhorn monitors and records health information for node disks. When disabled, disk health checks and status updates are skipped. nodeDiskHealthMonitoring: ~ # -- Comma-separated list of topology keys that the Longhorn CSI driver is allowed to pass through. When empty (default), no topology keys are passed through, and PVs will have no nodeAffinity. When configured (e.g., "topology.kubernetes.io/zone,topology.kubernetes.io/region"), only the specified keys are kept in topology segments. All other keys are filtered out from both CreateVolumeResponse.AccessibleTopology and NodeGetInfo topology. csiAllowedTopologyKeys: ~ # -- Setting that allows you to update the default backupstore. defaultBackupStore: # -- Endpoint used to access the default backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") backupTarget: ~ # -- Name of the Kubernetes secret associated with the default backup target. backupTargetCredentialSecret: ~ # -- Number of seconds that Longhorn waits before checking the default backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. pollInterval: ~ privateRegistry: # -- Set to `true` to automatically create a new private registry secret. createSecret: ~ # -- URL of a private registry. When unspecified, Longhorn uses the default system registry. registryUrl: ~ # -- User account used for authenticating with a private registry. registryUser: ~ # -- Password for authenticating with a private registry. registryPasswd: ~ # -- If create a new private registry secret is true, create a Kubernetes secret with this name; else use the existing secret of this name. Use it to pull images from your private registry. registrySecret: ~ longhornManager: log: # -- Format of Longhorn Manager logs. (Options: "plain", "json") format: plain # -- PriorityClass for Longhorn Manager. priorityClass: *defaultPriorityClassNameRef # -- Toleration for Longhorn Manager on nodes allowed to run Longhorn components. tolerations: [] ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above ## and uncomment this example block # - key: "key" # operator: "Equal" # value: "value" # effect: "NoSchedule" # -- Resource requests and limits for Longhorn Manager pods. resources: ~ # -- Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager. nodeSelector: {} ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above ## and uncomment this example block # label-key1: "label-value1" # label-key2: "label-value2" # -- Annotation for the Longhorn Manager service. serviceAnnotations: {} ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above ## and uncomment this example block # annotation-key1: "annotation-value1" # annotation-key2: "annotation-value2" serviceLabels: {} ## If you want to set labels for the Longhorn Manager service, delete the `{}` in the line above ## and uncomment this example block # label-key1: "label-value1" # label-key2: "label-value2" ## DaemonSet update strategy. Default "100% unavailable" matches the upgrade ## flow (old managers removed before new start); override for rolling updates ## if you prefer that behavior. updateStrategy: rollingUpdate: maxUnavailable: "100%" longhornDriver: log: # -- Format of longhorn-driver logs. (Options: "plain", "json") format: plain # -- PriorityClass for Longhorn Driver. priorityClass: *defaultPriorityClassNameRef # -- Toleration for Longhorn Driver on nodes allowed to run Longhorn components. tolerations: [] ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above ## and uncomment this example block # - key: "key" # operator: "Equal" # value: "value" # effect: "NoSchedule" # -- Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver. nodeSelector: {} ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above ## and uncomment this example block # label-key1: "label-value1" # label-key2: "label-value2" longhornUI: # -- Replica count for Longhorn UI. replicas: 2 # -- PriorityClass for Longhorn UI. priorityClass: *defaultPriorityClassNameRef # -- Affinity for Longhorn UI pods. Specify the affinity you want to use for Longhorn UI. affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - longhorn-ui topologyKey: kubernetes.io/hostname # -- Toleration for Longhorn UI on nodes allowed to run Longhorn components. tolerations: [] ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above ## and uncomment this example block # - key: "key" # operator: "Equal" # value: "value" # effect: "NoSchedule" # -- Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI. nodeSelector: {} ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above ## and uncomment this example block # label-key1: "label-value1" # label-key2: "label-value2" ingress: # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service. enabled: false # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller. # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases. ingressClassName: ~ # -- Hostname of the Layer 7 load balancer. host: sslip.io # -- Extra hostnames for TLS (Subject Alternative Names - SAN). Used when you need multiple FQDNs for the same ingress. # Example: # extraHosts: # - longhorn.example.com # - longhorn-ui.internal.local extraHosts: [] # -- Setting that allows you to enable TLS on ingress records. tls: false # -- Setting that allows you to enable secure connections to the Longhorn UI service via port 443. secureBackends: false # -- TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records. tlsSecret: longhorn.local-tls # -- Default ingress path. You can access the Longhorn UI by following the full ingress path {{host}}+{{path}}. path: / # -- Ingress path type. To maintain backward compatibility, the default value is "ImplementationSpecific". pathType: ImplementationSpecific ## If you're using kube-lego, you will want to add: ## kubernetes.io/tls-acme: true ## ## For a full list of possible ingress annotations, please see ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md ## ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set # -- Ingress annotations in the form of key-value pairs. annotations: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: true # -- Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses. secrets: ## If you're providing your own certificates, please use this to add the certificates as secrets ## key and certificate should start with -----BEGIN CERTIFICATE----- or ## -----BEGIN RSA PRIVATE KEY----- ## ## name should line up with a tlsSecret set further up ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set ## ## It is also possible to create and manage the certificates outside of this helm chart ## Please see README.md for more information # - name: longhorn.local-tls # key: # certificate: httproute: # -- Setting that allows Longhorn to generate HTTPRoute records for the Longhorn UI service using Gateway API. enabled: false # -- Gateway references for HTTPRoute. Specify which Gateway(s) should handle this route. parentRefs: [] ## Example: # - name: gateway-name # namespace: gateway-namespace # # Optional fields with defaults: # # group: gateway.networking.k8s.io # default # # kind: Gateway # default # # sectionName: https # optional, targets a specific listener # -- List of hostnames for the HTTPRoute. Multiple hostnames are supported. hostnames: [] ## Example: # - longhorn.example.com # - longhorn.example.org # -- Default path for HTTPRoute. You can access the Longhorn UI by following the full path. path: / # -- Path match type for HTTPRoute. (Options: "Exact", "PathPrefix") pathType: PathPrefix # -- Annotations for the HTTPRoute resource in the form of key-value pairs. annotations: {} ## Example: # annotation-key1: "annotation-value1" # -- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. enablePSP: false # -- Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`. namespaceOverride: "" # -- Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. annotations: {} serviceAccount: # -- Annotations to add to the service account annotations: {} metrics: serviceMonitor: # -- Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components. enabled: false # -- Additional labels for the Prometheus ServiceMonitor resource. additionalLabels: {} # -- Annotations for the Prometheus ServiceMonitor resource. annotations: {} # -- Interval at which Prometheus scrapes the metrics from the target. interval: "" # -- Timeout after which Prometheus considers the scrape to be failed. scrapeTimeout: "" # -- Configures the relabeling rules to apply the target’s metadata labels. See the [Prometheus Operator # documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for # formatting details. relabelings: [] # -- Configures the relabeling rules to apply to the samples before ingestion. See the [Prometheus Operator # documentation](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.Endpoint) for # formatting details. metricRelabelings: [] ## openshift settings openshift: # -- Setting that allows Longhorn to integrate with OpenShift. enabled: false ui: # -- Route for connections between Longhorn and the OpenShift web console. route: "longhorn-ui" # -- Port for accessing the OpenShift web console. port: 443 # -- Port for proxy that provides access to the OpenShift web console. proxy: 8443 # -- Setting that allows Longhorn to generate code coverage profiles. enableGoCoverDir: false # -- Add extra objects manifests extraObjects: [] ================================================ FILE: deploy/backupstores/README.md ================================================ # Longhorn Backupstores (Kustomize + Dynamic Credentials) This directory provides manifest templates to deploy **test-ready backup targets** for Longhorn, including: - Azurite (Azure Blob-compatible) - CIFS (SMB mount) - MinIO (S3-compatible object storage) All secrets are generated **dynamically** at runtime using Kustomize patches, to avoid committing hardcoded credentials. --- ## Purpose For security concerns related to credential exposure, we avoid static credential exposure in Git. Instead, we generate the Kustomize patches dynamically through a script to simulate test credentials without committing them. --- ## Directory Structure > Note: `overlays/` will be generated by `generate-backupstore-credentials.sh` ``` deploy/backupstores/ ├── base │   ├── azurite │   │   ├── azurite-backupstore.yaml │   │   └── kustomization.yaml │   ├── cifs │   │   ├── cifs-backupstore.yaml │   │   └── kustomization.yaml │   ├── minio │   │   ├── kustomization.yaml │   │   └── minio-backupstore.yaml │   └── nfs │   ├── kustomization.yaml │   └── nfs-backupstore.yaml ├── overlays │   └── generated-credentials │   ├── all │   │   └── kustomization.yaml │   ├── azurite │   │   ├── azurite-backupstore-secret-patch-longhorn-system.yaml │   │   └── kustomization.yaml │   ├── cifs │   │   ├── cifs-backupstore-secret-patch-default.yaml │   │   ├── cifs-backupstore-secret-patch-longhorn-system.yaml │   │   └── kustomization.yaml │   ├── minio │   │   ├── kustomization.yaml │   │   ├── minio-backupstore-secret-patch-default.yaml │   │   └── minio-backupstore-secret-patch-longhorn-system.yaml │   └── nfs │   └── kustomization.yaml └── README.md ``` --- ## Script: `generate-backupstore-credentials.sh` This script generates dummy secret patches for one or more backends. It supports two namespaces: `default` and `longhorn-system`. ⚠️ Be careful not to double-encode values. If using `--no-encode`, ensure your environment variables are already base64-encoded exactly once. ### Preconfigured credentials (edit as needed) > **Credential Notes:** > > - Kubernetes secrets require base64-encoded values. > - The script supports both: > - **Plaintext input (default)**: The script will automatically base64-encode values before writing them into Secret manifests. > - **Base64-encoded input**: Use the `--no-encode` flag to use your values as-is without re-encoding. In `generate-backupstore-credentials.sh`: ```bash # Azurite export AZBLOB_ACCOUNT_NAME="example-azblob-account" export AZBLOB_ACCOUNT_KEY="example-azblob-key" export AZBLOB_ENDPOINT="http://azblob-service.default:10000" # CIFS export CIFS_USERNAME="example-cifs-username" export CIFS_PASSWORD="example-cifs-password" # MinIO export AWS_ACCESS_KEY_ID="example-minio-access-key" export AWS_SECRET_ACCESS_KEY="example-minio-secret-key" export AWS_ENDPOINTS="https://minio-service.default:9000" export AWS_CERT="example-base64-cert" export AWS_CERT_KEY="example-base64-cert-key" ``` ### Options - ``: One of `azurite`, `cifs`, `minio`, `nfs` or `all`. Required. - `--no-encode` (optional): Indicates that the provided values are already base64-encoded and should be used as-is without additional encoding. ### Usage **Generate credentials for a single backend** ```bash ./scripts/generate-backupstore-credentials.sh minio ``` **Generate credentials for all backends** ```bash ./scripts/generate-backupstore-credentials.sh all ``` **By default, secrets are base64-encoded. Use --no-encode if the input is already base64.** ```bash ./scripts/generate-backupstore-credentials.sh minio --no-encode ``` ### Deploy / Delete **Deploy / Delete one backend** ```bash kubectl apply -k deploy/backupstores/overlays/generated-credentials/minio ``` ```bash kubectl delete -k deploy/backupstores/overlays/generated-credentials/minio ``` **Deploy / Delete all** ```bash kubectl apply -k deploy/backupstores/overlays/generated-credentials/all ``` ```bash kubectl delete -k deploy/backupstores/overlays/generated-credentials/all ``` ================================================ FILE: deploy/backupstores/base/azurite/azurite-backupstore.yaml ================================================ # same secret for longhorn-system namespace apiVersion: v1 kind: Secret metadata: name: azblob-secret namespace: longhorn-system type: Opaque data: {} --- apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-test-azblob namespace: default labels: app: longhorn-test-azblob spec: replicas: 1 selector: matchLabels: app: longhorn-test-azblob template: metadata: labels: app: longhorn-test-azblob spec: containers: - name: azurite image: mcr.microsoft.com/azure-storage/azurite:3.33.0 ports: - containerPort: 10000 --- apiVersion: v1 kind: Service metadata: name: azblob-service namespace: default spec: selector: app: longhorn-test-azblob ports: - port: 10000 targetPort: 10000 protocol: TCP sessionAffinity: ClientIP ================================================ FILE: deploy/backupstores/base/azurite/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - azurite-backupstore.yaml ================================================ FILE: deploy/backupstores/base/cifs/cifs-backupstore.yaml ================================================ apiVersion: v1 kind: Secret metadata: name: cifs-secret namespace: longhorn-system type: Opaque data: {} --- apiVersion: v1 kind: Secret metadata: name: cifs-secret namespace: default type: Opaque data: {} --- apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-test-cifs namespace: default labels: app: longhorn-test-cifs spec: replicas: 1 selector: matchLabels: app: longhorn-test-cifs template: metadata: labels: app: longhorn-test-cifs spec: volumes: - name: cifs-volume emptyDir: {} containers: - name: longhorn-test-cifs-container image: chanow/samba:latest ports: - containerPort: 139 - containerPort: 445 imagePullPolicy: Always env: - name: EXPORT_PATH value: /opt/backupstore - name: CIFS_DISK_IMAGE_SIZE_MB value: "8192" - name: CIFS_USERNAME valueFrom: secretKeyRef: name: cifs-secret key: CIFS_USERNAME - name: CIFS_PASSWORD valueFrom: secretKeyRef: name: cifs-secret key: CIFS_PASSWORD securityContext: privileged: true capabilities: add: ["SYS_ADMIN", "DAC_READ_SEARCH"] volumeMounts: - name: cifs-volume mountPath: "/opt/backupstore" args: ["-u", "$(CIFS_USERNAME);$(CIFS_PASSWORD)", "-s", "backupstore;$(EXPORT_PATH);yes;no;no;all;none"] --- kind: Service apiVersion: v1 metadata: name: longhorn-test-cifs-svc namespace: default spec: selector: app: longhorn-test-cifs clusterIP: None ports: - name: netbios-port port: 139 targetPort: 139 - name: microsoft-port port: 445 targetPort: 445 ================================================ FILE: deploy/backupstores/base/cifs/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cifs-backupstore.yaml ================================================ FILE: deploy/backupstores/base/minio/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - minio-backupstore.yaml ================================================ FILE: deploy/backupstores/base/minio/minio-backupstore.yaml ================================================ apiVersion: v1 kind: Secret metadata: name: minio-secret namespace: default type: Opaque data: {} --- # same secret for longhorn-system namespace apiVersion: v1 kind: Secret metadata: name: minio-secret namespace: longhorn-system type: Opaque data: {} --- apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-test-minio namespace: default labels: app: longhorn-test-minio spec: replicas: 1 selector: matchLabels: app: longhorn-test-minio template: metadata: labels: app: longhorn-test-minio spec: volumes: - name: minio-volume emptyDir: {} - name: minio-certificates secret: secretName: minio-secret items: - key: AWS_CERT path: public.crt - key: AWS_CERT_KEY path: private.key containers: - name: minio image: minio/minio:RELEASE.2022-02-01T18-00-14Z command: ["sh", "-c", "mkdir -p /storage/backupbucket && mkdir -p /root/.minio/certs && ln -s /root/certs/private.key /root/.minio/certs/private.key && ln -s /root/certs/public.crt /root/.minio/certs/public.crt && exec minio server /storage"] env: - name: MINIO_ROOT_USER valueFrom: secretKeyRef: name: minio-secret key: AWS_ACCESS_KEY_ID - name: MINIO_ROOT_PASSWORD valueFrom: secretKeyRef: name: minio-secret key: AWS_SECRET_ACCESS_KEY ports: - containerPort: 9000 volumeMounts: - name: minio-volume mountPath: "/storage" - name: minio-certificates mountPath: "/root/certs" readOnly: true --- apiVersion: v1 kind: Service metadata: name: minio-service namespace: default spec: selector: app: longhorn-test-minio ports: - port: 9000 targetPort: 9000 protocol: TCP sessionAffinity: ClientIP ================================================ FILE: deploy/backupstores/base/nfs/kustomization.yaml ================================================ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - nfs-backupstore.yaml ================================================ FILE: deploy/backupstores/base/nfs/nfs-backupstore.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-test-nfs namespace: default labels: app: longhorn-test-nfs spec: selector: matchLabels: app: longhorn-test-nfs template: metadata: labels: app: longhorn-test-nfs spec: volumes: - name: nfs-volume emptyDir: {} - name: ganesha-data emptyDir: {} containers: - name: longhorn-test-nfs-container image: longhornio/nfs-backupstore:latest imagePullPolicy: Always env: - name: EXPORT_ID value: "14" - name: EXPORT_PATH value: /opt/backupstore - name: PSEUDO_PATH value: /opt/backupstore - name: NFS_DISK_IMAGE_SIZE_MB value: "4096" command: ["bash", "-c", "chmod 700 /opt/backupstore && /opt/start_nfs.sh | tee /var/log/ganesha.log"] securityContext: privileged: true capabilities: add: ["SYS_ADMIN", "DAC_READ_SEARCH"] volumeMounts: - name: nfs-volume mountPath: "/opt/backupstore" - name: ganesha-data mountPath: /usr/local/var/lib/nfs/ganesha livenessProbe: exec: command: ["bash", "-c", "grep \"No export entries found\" /var/log/ganesha.log > /dev/null 2>&1 ; [ $? -ne 0 ]"] initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 4 --- kind: Service apiVersion: v1 metadata: name: longhorn-test-nfs-svc namespace: default spec: selector: app: longhorn-test-nfs clusterIP: None ports: - name: notnecessary port: 1234 targetPort: 1234 ================================================ FILE: deploy/longhorn-images.txt ================================================ longhornio/backing-image-manager:master-head longhornio/longhorn-engine:master-head longhornio/longhorn-instance-manager:master-head longhornio/longhorn-manager:master-head longhornio/longhorn-share-manager:master-head longhornio/longhorn-ui:master-head longhornio/longhorn-cli:master-head longhornio/csi-attacher:v4.10.0-20251226 longhornio/csi-provisioner:v5.3.0-20251226 longhornio/csi-resizer:v2.0.0-20251226 longhornio/csi-snapshotter:v8.4.0-20251226 longhornio/csi-node-driver-registrar:v2.15.0-20251226 longhornio/livenessprobe:v2.17.0-20251226 longhornio/support-bundle-kit:v0.0.79 ================================================ FILE: deploy/longhorn-okd.yaml ================================================ --- # Builtin: "helm template" does not respect --create-namespace apiVersion: v1 kind: Namespace metadata: name: longhorn-system --- # Source: longhorn/templates/priorityclass.yaml apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: "longhorn-critical" labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" globalDefault: false preemptionPolicy: PreemptLowerPriority value: 1000000000 --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-service-account namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-ui-service-account namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev annotations: serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"longhorn-ui"}}' --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-support-bundle namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev --- # Source: longhorn/templates/default-resource.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-resource namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: default-resource.yaml: |- --- # Source: longhorn/templates/default-setting.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-setting namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: default-setting.yaml: |- priority-class: "longhorn-critical" disable-revision-counter: "{\"v1\":\"true\"}" --- # Source: longhorn/templates/storageclass.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-storageclass namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: storageclass.yaml: | kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn annotations: storageclass.kubernetes.io/is-default-class: "true" provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: "Delete" volumeBindingMode: Immediate parameters: numberOfReplicas: "3" staleReplicaTimeout: "30" fromBackup: "" fsType: "ext4" dataLocality: "disabled" unmapMarkSnapChainRemoved: "ignored" disableRevisionCounter: "true" dataEngine: "v1" backupTargetName: "default" --- # Source: longhorn/templates/crds.yaml # Generated crds.yaml from github.com/longhorn/longhorn-manager/k8s/pkg/apis and the crds.yaml will be copied to longhorn/longhorn chart/templates and cannot be directly used by kubectl apply. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimagedatasources.longhorn.io spec: group: longhorn.io names: kind: BackingImageDataSource listKind: BackingImageDataSourceList plural: backingimagedatasources shortNames: - lhbids singular: backingimagedatasource scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID of the provisioned backing image file jsonPath: .spec.uuid name: UUID type: string - description: The current state of the pod used to provision the backing image file from source jsonPath: .status.currentState name: State type: string - description: The data source type jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size jsonPath: .status.size name: Size type: string - description: The node the backing image file will be prepared on jsonPath: .spec.nodeID name: Node type: string - description: The disk the backing image file will be prepared on jsonPath: .spec.diskUUID name: DiskUUID type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageDataSource is where Longhorn stores backing image data source object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source properties: checksum: type: string diskPath: type: string diskUUID: type: string fileTransferred: type: boolean nodeID: type: string parameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string uuid: type: string type: object status: description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source properties: checksum: type: string currentState: type: string ip: type: string message: type: string ownerID: type: string progress: type: integer runningParameters: additionalProperties: type: string nullable: true type: object size: format: int64 type: integer storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimagemanagers.longhorn.io spec: group: longhorn.io names: kind: BackingImageManager listKind: BackingImageManagerList plural: backingimagemanagers shortNames: - lhbim singular: backingimagemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The current state of the manager jsonPath: .status.currentState name: State type: string - description: The image the manager pod will use jsonPath: .spec.image name: Image type: string - description: The node the manager is on jsonPath: .spec.nodeID name: Node type: string - description: The disk the manager is responsible for jsonPath: .spec.diskUUID name: DiskUUID type: string - description: The disk path the manager is using jsonPath: .spec.diskPath name: DiskPath type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageManager is where Longhorn stores backing image manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager properties: backingImages: additionalProperties: type: string type: object diskPath: type: string diskUUID: type: string image: type: string nodeID: type: string type: object status: description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImageFileMap: additionalProperties: properties: currentChecksum: type: string message: type: string name: type: string progress: type: integer realSize: format: int64 type: integer senderManagerAddress: type: string sendingReference: type: integer size: format: int64 type: integer state: type: string uuid: type: string virtualSize: format: int64 type: integer type: object nullable: true type: object currentState: type: string ip: type: string ownerID: type: string storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimages.longhorn.io spec: group: longhorn.io names: kind: BackingImage listKind: BackingImageList plural: backingimages shortNames: - lhbi singular: backingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID jsonPath: .status.uuid name: UUID type: string - description: The source of the backing image file data jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size in each disk jsonPath: .status.size name: Size type: string - description: The virtual size of the image (may be larger than file size) jsonPath: .status.virtualSize name: VirtualSize type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImage is where Longhorn stores backing image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageSpec defines the desired state of the Longhorn backing image properties: checksum: type: string dataEngine: default: v1 enum: - v1 - v2 type: string diskFileSpecMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string evictionRequested: type: boolean type: object type: object diskSelector: items: type: string type: array disks: additionalProperties: type: string description: Deprecated. We are now using DiskFileSpecMap to assign different spec to the file on different disks. type: object minNumberOfCopies: type: integer nodeSelector: items: type: string type: array secret: type: string secretNamespace: type: string sourceParameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string type: object status: description: BackingImageStatus defines the observed state of the Longhorn backing image status properties: checksum: type: string diskFileStatusMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string lastStateTransitionTime: type: string message: type: string progress: type: integer state: type: string type: object nullable: true type: object diskLastRefAtMap: additionalProperties: type: string nullable: true type: object ownerID: type: string realSize: description: Real size of image in bytes, which may be smaller than the size when the file is a sparse file. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer size: format: int64 type: integer uuid: type: string v2FirstCopyDisk: type: string v2FirstCopyStatus: description: It is pending -> in-progress -> ready/failed type: string virtualSize: description: Virtual size of image in bytes, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backupbackingimages.longhorn.io spec: group: longhorn.io names: kind: BackupBackingImage listKind: BackupBackingImageList plural: backupbackingimages shortNames: - lhbbi singular: backupbackingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The backing image name jsonPath: .status.backingImage name: BackingImage type: string - description: The backing image size jsonPath: .status.size name: Size type: string - description: The backing image backup upload finished time jsonPath: .status.backupCreatedAt name: BackupCreatedAt type: string - description: The backing image backup state jsonPath: .status.state name: State type: string - description: The last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupBackingImage is where Longhorn stores backing image backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupTargetName: description: The backup target name. nullable: true type: string labels: additionalProperties: type: string description: The labels of backing image backup. type: object syncRequestedAt: description: The time to request run sync the remote backing image backup. format: date-time nullable: true type: string userCreated: description: Is this CR created by user through API or UI. type: boolean required: - backingImage - userCreated type: object status: description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupCreatedAt: description: The backing image backup upload finished time. type: string checksum: description: The checksum of the backing image. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the backing image backup. type: string labels: additionalProperties: type: string description: The labels of backing image backup. nullable: true type: object lastSyncedAt: description: The last time that the backing image backup was synced with the remote backup target. format: date-time nullable: true type: string managerAddress: description: The address of the backing image manager that runs backing image backup. type: string messages: additionalProperties: type: string description: The error messages when listing or inspecting backing image backup. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this CR. type: string progress: description: The backing image backup progress. type: integer secret: description: Record the secret if this backup backing image is encrypted type: string secretNamespace: description: Record the secret namespace if this backup backing image is encrypted type: string size: description: The backing image size. format: int64 type: integer state: description: |- The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The backing image backup URL. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backups.longhorn.io spec: group: longhorn.io names: kind: Backup listKind: BackupList plural: backups shortNames: - lhb singular: backup scope: Namespaced versions: - additionalPrinterColumns: - description: The snapshot name jsonPath: .status.snapshotName name: SnapshotName type: string - description: The snapshot size jsonPath: .status.size name: SnapshotSize type: string - description: The snapshot creation time jsonPath: .status.snapshotCreatedAt name: SnapshotCreatedAt type: string - description: The backup target name jsonPath: .status.backupTargetName name: BackupTarget type: string - description: The backup state jsonPath: .status.state name: State type: string - description: The backup last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: Backup is where Longhorn stores backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupSpec defines the desired state of the Longhorn backup properties: backupBlockSize: description: The backup block size. 0 means the legacy default size 2MiB, and -1 indicate the block size is invalid. enum: - "-1" - "2097152" - "16777216" format: int64 type: string backupMode: description: |- The backup mode of this backup. Can be "full" or "incremental" enum: - full - incremental type: string labels: additionalProperties: type: string description: The labels of snapshot backup. type: object snapshotName: description: The snapshot name. type: string syncRequestedAt: description: The time to request run sync the remote backup. format: date-time nullable: true type: string type: object status: description: BackupStatus defines the observed state of the Longhorn backup properties: backupCreatedAt: description: The snapshot backup upload finished time. type: string backupTargetName: description: The backup target name. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the snapshot backup. type: string labels: additionalProperties: type: string description: The labels of snapshot backup. nullable: true type: object lastSyncedAt: description: The last time that the backup was synced with the remote backup target. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when calling longhorn engine on listing or inspecting backups. nullable: true type: object newlyUploadDataSize: description: Size in bytes of newly uploaded data type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup CR. type: string progress: description: The snapshot backup progress. type: integer reUploadedDataSize: description: Size in bytes of reuploaded data type: string replicaAddress: description: The address of the replica that runs snapshot backup. type: string size: description: The snapshot size. type: string snapshotCreatedAt: description: The snapshot creation time. type: string snapshotName: description: The snapshot name. type: string state: description: |- The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The snapshot backup URL. type: string volumeBackingImageName: description: The volume's backing image name. type: string volumeCreated: description: The volume creation time. type: string volumeName: description: The volume name. type: string volumeSize: description: The volume size. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backuptargets.longhorn.io spec: group: longhorn.io names: kind: BackupTarget listKind: BackupTargetList plural: backuptargets shortNames: - lhbt singular: backuptarget scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target URL jsonPath: .spec.backupTargetURL name: URL type: string - description: The backup target credential secret jsonPath: .spec.credentialSecret name: Credential type: string - description: The backup target poll interval jsonPath: .spec.pollInterval name: LastBackupAt type: string - description: Indicate whether the backup target is available or not jsonPath: .status.available name: Available type: boolean - description: The backup target last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupTarget is where Longhorn stores backup target object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupTargetSpec defines the desired state of the Longhorn backup target properties: backupTargetURL: description: The backup target URL. type: string credentialSecret: description: The backup target credential secret. type: string pollInterval: description: The interval that the cluster needs to run sync with the backup target. type: string syncRequestedAt: description: The time to request run sync the remote backup target. format: date-time nullable: true type: string type: object status: description: BackupTargetStatus defines the observed state of the Longhorn backup target properties: available: description: Available indicates if the remote backup target is available or not. type: boolean conditions: description: Records the reason on why the backup target is unavailable. items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array lastSyncedAt: description: The last time that the controller synced with the remote backup target. format: date-time nullable: true type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup target CR. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backupvolumes.longhorn.io spec: group: longhorn.io names: kind: BackupVolume listKind: BackupVolumeList plural: backupvolumes shortNames: - lhbv singular: backupvolume scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target name jsonPath: .spec.backupTargetName name: BackupTarget type: string - description: The backup volume creation time jsonPath: .status.createdAt name: CreatedAt type: string - description: The backup volume last backup name jsonPath: .status.lastBackupName name: LastBackupName type: string - description: The backup volume last backup time jsonPath: .status.lastBackupAt name: LastBackupAt type: string - description: The backup volume last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupVolume is where Longhorn stores backup volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupVolumeSpec defines the desired state of the Longhorn backup volume properties: backupTargetName: description: The backup target name that the backup volume was synced. nullable: true type: string syncRequestedAt: description: The time to request run sync the remote backup volume. format: date-time nullable: true type: string volumeName: description: The volume name that the backup volume was used to backup. type: string type: object status: description: BackupVolumeStatus defines the observed state of the Longhorn backup volume properties: backingImageChecksum: description: the backing image checksum. type: string backingImageName: description: The backing image name. type: string createdAt: description: The backup volume creation time. type: string dataStored: description: The backup volume block count. type: string labels: additionalProperties: type: string description: The backup volume labels. nullable: true type: object lastBackupAt: description: The latest volume backup time. type: string lastBackupName: description: The latest volume backup name. type: string lastModificationTime: description: The backup volume config last modification time. format: date-time nullable: true type: string lastSyncedAt: description: The last time that the backup volume was synced into the cluster. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when call longhorn engine on list or inspect backup volumes. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this backup volume CR. type: string size: description: The backup volume size. type: string storageClassName: description: the storage class name of pv/pvc binding with the volume. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: engineimages.longhorn.io spec: group: longhorn.io names: kind: EngineImage listKind: EngineImageList plural: engineimages shortNames: - lhei singular: engineimage scope: Namespaced versions: - additionalPrinterColumns: - description: Compatibility of the engine image jsonPath: .status.incompatible name: Incompatible type: boolean - description: State of the engine image jsonPath: .status.state name: State type: string - description: The Longhorn engine image jsonPath: .spec.image name: Image type: string - description: Number of resources using the engine image jsonPath: .status.refCount name: RefCount type: integer - description: The build date of the engine image jsonPath: .status.buildDate name: BuildDate type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: EngineImage is where Longhorn stores engine image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineImageSpec defines the desired state of the Longhorn engine image properties: image: minLength: 1 type: string required: - image type: object status: description: EngineImageStatus defines the observed state of the Longhorn engine image properties: buildDate: type: string cliAPIMinVersion: type: integer cliAPIVersion: type: integer conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array controllerAPIMinVersion: type: integer controllerAPIVersion: type: integer dataFormatMinVersion: type: integer dataFormatVersion: type: integer gitCommit: type: string incompatible: type: boolean noRefSince: type: string nodeDeploymentMap: additionalProperties: type: boolean nullable: true type: object ownerID: type: string refCount: type: integer state: type: string version: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: engines.longhorn.io spec: group: longhorn.io names: kind: Engine listKind: EngineList plural: engines shortNames: - lhe singular: engine scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the engine jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the engine jsonPath: .status.currentState name: State type: string - description: The node that the engine is on jsonPath: .spec.nodeID name: Node type: string - description: The instance manager of the engine jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the engine jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Engine is where Longhorn stores engine object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineSpec defines the desired state of the Longhorn engine properties: active: type: boolean backupVolume: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string disableFrontend: type: boolean frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string logRequested: type: boolean nodeID: type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. maximum: 5 minimum: 0 type: integer replicaAddressMap: additionalProperties: type: string type: object requestedBackupRestore: type: string requestedDataSource: type: string revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemovedEnabled: type: boolean upgradedReplicaAddressMap: additionalProperties: type: string type: object volumeName: type: string volumeSize: format: int64 type: string type: object status: description: EngineStatus defines the observed state of the Longhorn engine properties: backupStatus: additionalProperties: properties: backupURL: type: string error: type: string progress: type: integer replicaAddress: type: string snapshotName: type: string state: type: string type: object nullable: true type: object cloneStatus: additionalProperties: properties: error: type: string fromReplicaAddress: type: string isCloning: type: boolean progress: type: integer snapshotName: type: string state: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentReplicaAddressMap: additionalProperties: type: string nullable: true type: object currentSize: format: int64 type: string currentState: type: string endpoint: type: string instanceManagerName: type: string ip: type: string isExpanding: type: boolean lastExpansionError: type: string lastExpansionFailedAt: type: string lastRestoredBackup: type: string logFetched: type: boolean ownerID: type: string port: type: integer purgeStatus: additionalProperties: properties: error: type: string isPurging: type: boolean progress: type: integer state: type: string type: object nullable: true type: object rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. minimum: 0 type: integer rebuildStatus: additionalProperties: properties: appliedRebuildingMBps: format: int64 type: integer error: type: string fromReplicaAddress: description: Deprecated. We are now using FromReplicaAddressList to list all source replicas. type: string fromReplicaAddressList: items: type: string type: array isRebuilding: type: boolean progress: type: integer state: type: string type: object nullable: true type: object replicaModeMap: additionalProperties: type: string nullable: true type: object replicaTransitionTimeMap: additionalProperties: type: string description: |- ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). type: object restoreStatus: additionalProperties: properties: backupURL: type: string currentRestoringBackup: type: string error: type: string filename: type: string isRestoring: type: boolean lastRestored: type: string progress: type: integer state: type: string type: object nullable: true type: object salvageExecuted: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string snapshots: additionalProperties: properties: children: additionalProperties: type: boolean nullable: true type: object created: type: string labels: additionalProperties: type: string nullable: true type: object name: type: string parent: type: string removed: type: boolean size: type: string usercreated: type: boolean type: object nullable: true type: object snapshotsError: type: string started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer unmapMarkSnapChainRemovedEnabled: type: boolean uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: instancemanagers.longhorn.io spec: group: longhorn.io names: kind: InstanceManager listKind: InstanceManagerList plural: instancemanagers shortNames: - lhim singular: instancemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the instance manager jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the instance manager jsonPath: .status.currentState name: State type: string - description: The type of the instance manager (engine or replica) jsonPath: .spec.type name: Type type: string - description: The node that the instance manager is running on jsonPath: .spec.nodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: InstanceManager is where Longhorn stores instance manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: InstanceManagerSpec defines the desired state of the Longhorn instance manager properties: dataEngine: type: string dataEngineSpec: properties: v2: properties: cpuMask: type: string type: object type: object image: type: string nodeID: type: string type: enum: - aio - engine - replica type: string type: object status: description: InstanceManagerStatus defines the observed state of the Longhorn instance manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImages: additionalProperties: properties: currentChecksum: type: string diskUUID: type: string message: type: string name: type: string progress: type: integer size: format: int64 type: integer state: type: string uuid: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentState: type: string dataEngineStatus: properties: v2: properties: cpuMask: type: string interruptModeEnabled: description: |- InterruptModeEnabled indicates whether the V2 data engine is running in interrupt mode (true) or polling mode (false). Set by Longhorn manager; read-only to users. enum: - "" - "true" - "false" type: string type: object type: object instanceEngines: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object instanceReplicas: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object ip: type: string ownerID: type: string proxyApiMinVersion: type: integer proxyApiVersion: type: integer type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: nodes.longhorn.io spec: group: longhorn.io names: kind: Node listKind: NodeList plural: nodes shortNames: - lhn singular: node scope: Namespaced versions: - additionalPrinterColumns: - description: Indicate whether the node is ready jsonPath: .status.conditions[?(@.type=='Ready')].status name: Ready type: string - description: Indicate whether the user disabled/enabled replica scheduling for the node jsonPath: .spec.allowScheduling name: AllowScheduling type: boolean - description: Indicate whether Longhorn can schedule replicas on the node jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Schedulable type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Node is where Longhorn stores Longhorn node object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: NodeSpec defines the desired state of the Longhorn node properties: allowScheduling: type: boolean disks: additionalProperties: properties: allowScheduling: type: boolean diskDriver: enum: - "" - auto - aio - nvme type: string diskType: enum: - filesystem - block type: string evictionRequested: type: boolean path: type: string storageReserved: format: int64 type: integer tags: items: type: string type: array type: object type: object evictionRequested: type: boolean instanceManagerCPURequest: type: integer name: type: string tags: items: type: string type: array type: object status: description: NodeStatus defines the observed state of the Longhorn node properties: autoEvicting: type: boolean conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskStatus: additionalProperties: properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskDriver: type: string diskName: type: string diskPath: type: string diskType: type: string diskUUID: type: string filesystemType: type: string healthData: additionalProperties: properties: attributes: items: properties: id: type: integer name: type: string rawString: type: string rawValue: format: int64 type: integer threshold: type: integer value: type: integer whenFailed: type: string worst: type: integer type: object type: array capacity: format: int64 type: integer diskName: type: string diskType: type: string firmwareVersion: type: string healthStatus: enum: - FAILED - PASSED - UNKNOWN - WARNING type: string modelName: type: string serialNumber: type: string source: enum: - SMART - SPDK type: string temperature: type: integer type: object type: object healthDataLastCollectedAt: format: date-time type: string instanceManagerName: type: string scheduledBackingImage: additionalProperties: format: int64 type: integer nullable: true type: object scheduledReplica: additionalProperties: format: int64 type: integer nullable: true type: object storageAvailable: format: int64 type: integer storageMaximum: format: int64 type: integer storageScheduled: format: int64 type: integer type: object nullable: true type: object region: type: string snapshotCheckStatus: properties: lastPeriodicCheckedAt: format: date-time type: string type: object zone: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: orphans.longhorn.io spec: group: longhorn.io names: kind: Orphan listKind: OrphanList plural: orphans shortNames: - lho singular: orphan scope: Namespaced versions: - additionalPrinterColumns: - description: The type of the orphan jsonPath: .spec.orphanType name: Type type: string - description: The node that the orphan is on jsonPath: .spec.nodeID name: Node type: string name: v1beta2 schema: openAPIV3Schema: description: Orphan is where Longhorn stores orphan object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: OrphanSpec defines the desired state of the Longhorn orphaned data properties: dataEngine: description: |- The type of data engine for instance orphan. Can be "v1", "v2". enum: - v1 - v2 type: string nodeID: description: The node ID on which the controller is responsible to reconcile this orphan CR. type: string orphanType: description: |- The type of the orphaned data. Can be "replica". type: string parameters: additionalProperties: type: string description: The parameters of the orphaned data type: object type: object status: description: OrphanStatus defines the observed state of the Longhorn orphaned data properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: recurringjobs.longhorn.io spec: group: longhorn.io names: kind: RecurringJob listKind: RecurringJobList plural: recurringjobs shortNames: - lhrj singular: recurringjob scope: Namespaced versions: - additionalPrinterColumns: - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume jsonPath: .spec.groups name: Groups type: string - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup" jsonPath: .spec.task name: Task type: string - description: The cron expression represents recurring job scheduling jsonPath: .spec.cron name: Cron type: string - description: The number of snapshots/backups to keep for the volume jsonPath: .spec.retain name: Retain type: integer - description: The concurrent job to run by each cron job jsonPath: .spec.concurrency name: Concurrency type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date - description: Specify the labels jsonPath: .spec.labels name: Labels type: string name: v1beta2 schema: openAPIV3Schema: description: RecurringJob is where Longhorn stores recurring job object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: RecurringJobSpec defines the desired state of the Longhorn recurring job properties: concurrency: description: The concurrency of taking the snapshot/backup. type: integer cron: description: The cron setting. type: string groups: description: The recurring job group. items: type: string type: array labels: additionalProperties: type: string description: The label of the snapshot/backup. type: object name: description: The recurring job name. type: string parameters: additionalProperties: type: string description: |- The parameters of the snapshot/backup. Support parameters: "full-backup-interval", "volume-backup-policy". type: object retain: description: The retain count of the snapshot/backup. type: integer task: description: |- The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup". enum: - snapshot - snapshot-force-create - snapshot-cleanup - snapshot-delete - backup - backup-force-create - filesystem-trim - system-backup type: string type: object status: description: RecurringJobStatus defines the observed state of the Longhorn recurring job properties: executionCount: description: The number of jobs that have been triggered. type: integer ownerID: description: The owner ID which is responsible to reconcile this recurring job CR. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: replicas.longhorn.io spec: group: longhorn.io names: kind: Replica listKind: ReplicaList plural: replicas shortNames: - lhr singular: replica scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the replica jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the replica jsonPath: .status.currentState name: State type: string - description: The node that the replica is on jsonPath: .spec.nodeID name: Node type: string - description: The disk that the replica is on jsonPath: .spec.diskID name: Disk type: string - description: The instance manager of the replica jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the replica jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Replica is where Longhorn stores replica object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ReplicaSpec defines the desired state of the Longhorn replica properties: active: type: boolean backingImage: type: string dataDirectoryName: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string diskID: type: string diskPath: type: string engineName: type: string evictionRequested: type: boolean failedAt: description: |- FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string hardNodeAffinity: type: string healthyAt: description: |- HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string image: type: string lastFailedAt: description: |- LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string lastHealthyAt: description: |- LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string logRequested: type: boolean migrationEngineName: description: |- MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only used for live migration of v2 data engine type: string nodeID: type: string rebuildRetryCount: type: integer revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string unmapMarkDiskChainRemovedEnabled: type: boolean volumeName: type: string volumeSize: format: int64 type: string type: object status: description: ReplicaStatus defines the observed state of the Longhorn replica properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentState: type: string instanceManagerName: type: string ip: type: string logFetched: type: boolean ownerID: type: string port: type: integer salvageExecuted: type: boolean started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: settings.longhorn.io spec: group: longhorn.io names: kind: Setting listKind: SettingList plural: settings shortNames: - lhs singular: setting scope: Namespaced versions: - additionalPrinterColumns: - description: The value of the setting jsonPath: .value name: Value type: string - description: The setting is applied jsonPath: .status.applied name: Applied type: boolean - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Setting is where Longhorn stores setting object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: description: The status of the setting. properties: applied: description: The setting is applied. type: boolean required: - applied type: object value: description: |- The value of the setting. - It can be a non-JSON formatted string that is applied to all the applicable data engines listed in the setting definition. - It can be a JSON formatted string that contains values for applicable data engines listed in the setting definition's Default. type: string required: - value type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: sharemanagers.longhorn.io spec: group: longhorn.io names: kind: ShareManager listKind: ShareManagerList plural: sharemanagers shortNames: - lhsm singular: sharemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the share manager jsonPath: .status.state name: State type: string - description: The node that the share manager is owned by jsonPath: .status.ownerID name: Node type: string - description: The current image of the share manager jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: ShareManager is where Longhorn stores share manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ShareManagerSpec defines the desired state of the Longhorn share manager properties: image: description: Share manager image used for creating a share manager pod type: string type: object status: description: ShareManagerStatus defines the observed state of the Longhorn share manager properties: currentImage: description: The image currently used by the share manager pod type: string endpoint: description: NFS endpoint that can access the mounted filesystem of the volume type: string ownerID: description: The node ID on which the controller is responsible to reconcile this share manager resource type: string state: description: The state of the share manager resource type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: snapshots.longhorn.io spec: group: longhorn.io names: kind: Snapshot listKind: SnapshotList plural: snapshots shortNames: - lhsnap singular: snapshot scope: Namespaced versions: - additionalPrinterColumns: - description: The volume that this snapshot belongs to jsonPath: .spec.volume name: Volume type: string - description: Timestamp when the point-in-time snapshot was taken jsonPath: .status.creationTime name: CreationTime type: string - description: Indicates if the snapshot is ready to be used to restore/backup a volume jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Represents the minimum size of volume required to rehydrate from this snapshot jsonPath: .status.restoreSize name: RestoreSize type: string - description: The actual size of the snapshot jsonPath: .status.size name: Size type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Snapshot is the Schema for the snapshots API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SnapshotSpec defines the desired state of Longhorn Snapshot properties: createSnapshot: description: require creating a new snapshot type: boolean labels: additionalProperties: type: string description: The labels of snapshot nullable: true type: object volume: description: |- the volume that this snapshot belongs to. This field is immutable after creation. type: string required: - volume type: object status: description: SnapshotStatus defines the observed state of Longhorn Snapshot properties: checksum: type: string checksumCalculatedAt: description: |- ChecksumCalculatedAt is the RFC3339 timestamp indicating when the checksum for this snapshot was last calculated or updated. type: string children: additionalProperties: type: boolean nullable: true type: object creationTime: type: string error: type: string labels: additionalProperties: type: string nullable: true type: object markRemoved: type: boolean ownerID: type: string parent: type: string readyToUse: type: boolean restoreSize: format: int64 type: integer size: format: int64 type: integer userCreated: type: boolean type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: supportbundles.longhorn.io spec: group: longhorn.io names: kind: SupportBundle listKind: SupportBundleList plural: supportbundles shortNames: - lhbundle singular: supportbundle scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the support bundle jsonPath: .status.state name: State type: string - description: The issue URL jsonPath: .spec.issueURL name: Issue type: string - description: A brief description of the issue jsonPath: .spec.description name: Description type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SupportBundle is where Longhorn stores support bundle object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle properties: description: description: A brief description of the issue type: string issueURL: description: The issue URL nullable: true type: string nodeID: description: The preferred responsible controller node ID. type: string required: - description type: object status: description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object type: array filename: type: string filesize: format: int64 type: integer image: description: The support bundle manager image type: string managerIP: description: The support bundle manager IP type: string ownerID: description: The current responsible controller node ID type: string progress: type: integer state: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: systembackups.longhorn.io spec: group: longhorn.io names: kind: SystemBackup listKind: SystemBackupList plural: systembackups shortNames: - lhsb singular: systembackup scope: Namespaced versions: - additionalPrinterColumns: - description: The system backup Longhorn version jsonPath: .status.version name: Version type: string - description: The system backup state jsonPath: .status.state name: State type: string - description: The system backup creation time jsonPath: .status.createdAt name: Created type: string - description: The last time that the system backup was synced into the cluster jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: SystemBackup is where Longhorn stores system backup object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup properties: volumeBackupPolicy: description: |- The create volume backup policy Can be "if-not-present", "always" or "disabled" nullable: true type: string type: object status: description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array createdAt: description: The system backup creation time. format: date-time type: string gitCommit: description: The saved Longhorn manager git commit. nullable: true type: string lastSyncedAt: description: The last time that the system backup was synced into the cluster. format: date-time nullable: true type: string managerImage: description: The saved manager image. type: string ownerID: description: The node ID of the responsible controller to reconcile this SystemBackup. type: string state: description: The system backup state. type: string version: description: The saved Longhorn version. nullable: true type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: systemrestores.longhorn.io spec: group: longhorn.io names: kind: SystemRestore listKind: SystemRestoreList plural: systemrestores shortNames: - lhsr singular: systemrestore scope: Namespaced versions: - additionalPrinterColumns: - description: The system restore state jsonPath: .status.state name: State type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SystemRestore is where Longhorn stores system restore object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore properties: systemBackup: description: The system backup name in the object store. type: string required: - systemBackup type: object status: description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: description: The node ID of the responsible controller to reconcile this SystemRestore. type: string sourceURL: description: The source system backup URL. type: string state: description: The system restore state. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: volumeattachments.longhorn.io spec: group: longhorn.io names: kind: VolumeAttachment listKind: VolumeAttachmentList plural: volumeattachments shortNames: - lhva singular: volumeattachment scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: VolumeAttachment stores attachment information of a Longhorn volume properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment properties: attachmentTickets: additionalProperties: properties: generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string nodeID: description: The node that this attachment is requesting type: string parameters: additionalProperties: type: string description: Optional additional parameter for this attachment type: object type: type: string type: object type: object volume: description: The name of Longhorn volume of this VolumeAttachment type: string required: - volume type: object status: description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment properties: attachmentTicketStatuses: additionalProperties: properties: conditions: description: Record any error when trying to fulfill this attachment items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string satisfied: description: Indicate whether this attachment ticket has been satisfied type: boolean required: - conditions - satisfied type: object type: object type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: volumes.longhorn.io spec: group: longhorn.io names: kind: Volume listKind: VolumeList plural: volumes shortNames: - lhv singular: volume scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the volume jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the volume jsonPath: .status.state name: State type: string - description: The robustness of the volume jsonPath: .status.robustness name: Robustness type: string - description: The scheduled condition of the volume jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Scheduled type: string - description: The size of the volume jsonPath: .spec.size name: Size type: string - description: The node that the volume is currently attaching to jsonPath: .status.currentNodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Volume is where Longhorn stores volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeSpec defines the desired state of the Longhorn volume properties: Standby: type: boolean accessMode: enum: - rwo - rwop - rwx type: string backingImage: type: string x-kubernetes-validations: - message: BackingImage is immutable rule: self == oldSelf backupBlockSize: description: BackupBlockSize indicate the block size to create backups. The block size is immutable. enum: - "2097152" - "16777216" format: int64 type: string backupCompressionMethod: enum: - none - lz4 - gzip type: string backupTargetName: description: The backup target name that the volume will be backed up to or is synced. type: string cloneMode: enum: - "" - full-copy - linked-clone type: string dataEngine: enum: - v1 - v2 type: string dataLocality: enum: - disabled - best-effort - strict-local type: string dataSource: type: string disableFrontend: type: boolean diskSelector: items: type: string type: array encrypted: type: boolean x-kubernetes-validations: - message: Encrypted is immutable rule: self == oldSelf freezeFilesystemForSnapshot: description: Setting that freezes the filesystem on the root partition before a snapshot is created. enum: - ignored - enabled - disabled type: string fromBackup: type: string frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string lastAttachedBy: type: string migratable: type: boolean migrationNodeID: type: string nodeID: type: string nodeSelector: items: type: string type: array numberOfReplicas: type: integer offlineRebuilding: description: |- Specifies whether Longhorn should rebuild replicas while the detached volume is degraded. - ignored: Use the global setting for offline replica rebuilding. - enabled: Enable offline rebuilding for this volume, regardless of the global setting. - disabled: Disable offline rebuilding for this volume, regardless of the global setting enum: - ignored - disabled - enabled type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. When set to 0, it means following the global setting. maximum: 5 minimum: 0 type: integer replicaAutoBalance: enum: - ignored - disabled - least-effort - best-effort type: string replicaDiskSoftAntiAffinity: description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. enum: - ignored - enabled - disabled type: string replicaRebuildingBandwidthLimit: description: ReplicaRebuildingBandwidthLimit controls the maximum write bandwidth (in megabytes per second) allowed on the destination replica during the rebuilding process. Set this value to 0 to disable bandwidth limiting. format: int64 minimum: 0 type: integer replicaSoftAntiAffinity: description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. enum: - ignored - enabled - disabled type: string replicaZoneSoftAntiAffinity: description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. enum: - ignored - enabled - disabled type: string restoreVolumeRecurringJob: enum: - ignored - enabled - disabled type: string revisionCounterDisabled: type: boolean size: format: int64 type: string snapshotDataIntegrity: enum: - ignored - disabled - enabled - fast-check type: string snapshotHashingRequestedAt: description: |- SnapshotHashingRequestedAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when an on-demand snapshot checksum calculation is requested. When this value is set and is later than LastOnDemandSnapshotHashingCompleteAt, the system will calculate checksums for all user snapshots. If SnapshotHashingRequestedAt differs from LastOnDemandSnapshotHashingCompleteAt, it indicates that a hashing request is still in progress, and a new request will be rejected. type: string snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string staleReplicaTimeout: type: integer ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemoved: enum: - ignored - disabled - enabled type: string type: object status: description: VolumeStatus defines the observed state of the Longhorn volume properties: actualSize: format: int64 type: integer cloneStatus: properties: attemptCount: type: integer nextAllowedAttemptAt: type: string snapshot: type: string sourceVolume: type: string state: type: string type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentMigrationNodeID: description: the node that this volume is currently migrating to type: string currentNodeID: type: string expansionRequired: type: boolean frontendDisabled: type: boolean isStandby: type: boolean kubernetesStatus: properties: lastPVCRefAt: type: string lastPodRefAt: type: string namespace: description: determine if PVC/Namespace is history or not type: string pvName: type: string pvStatus: type: string pvcName: type: string workloadsStatus: description: determine if Pod/Workload is history or not items: properties: podName: type: string podStatus: type: string workloadName: type: string workloadType: type: string type: object nullable: true type: array type: object lastBackup: type: string lastBackupAt: type: string lastDegradedAt: type: string lastOnDemandSnapshotHashingCompleteAt: description: |- LastOnDemandSnapshotHashingCompleteAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when the most recent on-demand snapshot checksum calculation completed. When this value matches SnapshotHashingRequestedAt, the requested on-demand checksum calculation is considered complete. type: string ownerID: type: string remountRequestedAt: type: string restoreInitiated: type: boolean restoreRequired: type: boolean robustness: type: string shareEndpoint: type: string shareState: type: string state: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-role labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev rules: - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - "*" - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "delete", "deletecollection"] - apiGroups: [""] resources: ["secrets", "services", "endpoints", "configmaps", "serviceaccounts", "pods/log"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status", "nodes"] verbs: ["*"] - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "statefulsets", "deployments", "replicasets"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "list", "watch"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["watch", "list"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "volumeattributesclasses", "csinodes", "csidrivers", "csistoragecapacities"] verbs: ["*"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] verbs: ["*"] - apiGroups: ["longhorn.io"] resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status", "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", "engineimages/finalizers", "nodes/finalizers", "instancemanagers/finalizers", "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch"] - apiGroups: ["metrics.k8s.io"] resources: ["pods", "nodes"] verbs: ["get", "list"] - apiGroups: ["apiregistration.k8s.io"] resources: ["apiservices"] verbs: ["list", "watch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] verbs: ["get", "list", "create", "patch", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterrolebindings", "clusterroles"] verbs: ["*"] --- # Source: longhorn/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-ocp-privileged-role labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev rules: - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] resourceNames: ["anyuid", "privileged"] verbs: ["use"] --- # Source: longhorn/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-bind labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system --- # Source: longhorn/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-support-bundle labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: longhorn-support-bundle namespace: longhorn-system --- # Source: longhorn/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-ocp-privileged-bind labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-ocp-privileged-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system - kind: ServiceAccount name: longhorn-ui-service-account namespace: longhorn-system - kind: ServiceAccount name: default # supportbundle-agent-support-bundle uses default sa namespace: longhorn-system --- # Source: longhorn/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: longhorn namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev rules: - apiGroups: [""] resources: ["pods", "pods/log", "events", "secrets", "services", "endpoints", "configmaps", "serviceaccounts", "persistentvolumeclaims", "persistentvolumeclaims/status"] verbs: ["*"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "statefulsets", "replicasets"] verbs: ["*"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["*"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["*"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["*"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["*"] --- # Source: longhorn/templates/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: longhorn namespace: longhorn-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: longhorn subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system --- # Source: longhorn/templates/daemonset-sa.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager name: longhorn-backend namespace: longhorn-system spec: type: ClusterIP selector: app: longhorn-manager ports: - name: manager port: 9500 targetPort: manager --- # Source: longhorn/templates/deployment-ui.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-ui namespace: longhorn-system annotations: service.alpha.openshift.io/serving-cert-secret-name: longhorn-ui-tls spec: ports: - name: longhorn-ui port: 443 targetPort: 8443 selector: app: longhorn-ui --- # Source: longhorn/templates/deployment-ui.yaml kind: Service apiVersion: v1 metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-frontend namespace: longhorn-system spec: type: ClusterIP selector: app: longhorn-ui ports: - name: http port: 80 targetPort: http nodePort: null --- # Source: longhorn/templates/services.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-admission-webhook name: longhorn-admission-webhook namespace: longhorn-system spec: type: ClusterIP selector: longhorn.io/admission-webhook: longhorn-admission-webhook ports: - name: admission-webhook port: 9502 targetPort: admission-wh --- # Source: longhorn/templates/services.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-recovery-backend name: longhorn-recovery-backend namespace: longhorn-system spec: type: ClusterIP selector: longhorn.io/recovery-backend: longhorn-recovery-backend ports: - name: recovery-backend port: 9503 targetPort: recov-backend --- # Source: longhorn/templates/daemonset-sa.yaml apiVersion: apps/v1 kind: DaemonSet metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager name: longhorn-manager namespace: longhorn-system spec: selector: matchLabels: app: longhorn-manager template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager spec: containers: - name: longhorn-manager image: docker.io/longhornio/longhorn-manager:master-head imagePullPolicy: IfNotPresent securityContext: privileged: true command: - longhorn-manager - -d - daemon - --engine-image - "docker.io/longhornio/longhorn-engine:master-head" - --instance-manager-image - "docker.io/longhornio/longhorn-instance-manager:master-head" - --share-manager-image - "docker.io/longhornio/longhorn-share-manager:master-head" - --backing-image-manager-image - "docker.io/longhornio/backing-image-manager:master-head" - --support-bundle-manager-image - "docker.io/longhornio/support-bundle-kit:v0.0.79" - --manager-image - "docker.io/longhornio/longhorn-manager:master-head" - --service-account - longhorn-service-account - --upgrade-version-check ports: - containerPort: 9500 name: manager - containerPort: 9502 name: admission-wh - containerPort: 9503 name: recov-backend readinessProbe: httpGet: path: /v1/healthz port: 9502 scheme: HTTPS volumeMounts: - name: boot mountPath: /host/boot/ readOnly: true - name: dev mountPath: /host/dev/ - name: proc mountPath: /host/proc/ readOnly: true - name: etc mountPath: /host/etc/ readOnly: true - name: longhorn mountPath: /var/lib/longhorn/ mountPropagation: Bidirectional - name: longhorn-grpc-tls mountPath: /tls-files/ env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: pre-pull-share-manager-image imagePullPolicy: IfNotPresent image: docker.io/longhornio/longhorn-share-manager:master-head command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"] volumes: - name: boot hostPath: path: /boot/ - name: dev hostPath: path: /dev/ - name: proc hostPath: path: /proc/ - name: etc hostPath: path: /etc/ - name: longhorn hostPath: path: /var/lib/longhorn/ - name: longhorn-grpc-tls secret: secretName: longhorn-grpc-tls optional: true priorityClassName: "longhorn-critical" serviceAccountName: longhorn-service-account updateStrategy: rollingUpdate: maxUnavailable: 100% --- # Source: longhorn/templates/deployment-driver.yaml apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-driver-deployer namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev spec: replicas: 1 selector: matchLabels: app: longhorn-driver-deployer template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-driver-deployer spec: initContainers: - name: wait-longhorn-manager image: docker.io/longhornio/longhorn-manager:master-head command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer image: docker.io/longhornio/longhorn-manager:master-head imagePullPolicy: IfNotPresent command: - longhorn-manager - -d - deploy-driver - --manager-image - "docker.io/longhornio/longhorn-manager:master-head" - --manager-url - http://longhorn-backend:9500/v1 env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName - name: CSI_ATTACHER_IMAGE value: "docker.io/longhornio/csi-attacher:v4.10.0-20251226" - name: CSI_PROVISIONER_IMAGE value: "docker.io/longhornio/csi-provisioner:v5.3.0-20251226" - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE value: "docker.io/longhornio/csi-node-driver-registrar:v2.15.0-20251226" - name: CSI_RESIZER_IMAGE value: "docker.io/longhornio/csi-resizer:v2.0.0-20251226" - name: CSI_SNAPSHOTTER_IMAGE value: "docker.io/longhornio/csi-snapshotter:v8.4.0-20251226" - name: CSI_LIVENESS_PROBE_IMAGE value: "docker.io/longhornio/livenessprobe:v2.17.0-20251226" priorityClassName: "longhorn-critical" serviceAccountName: longhorn-service-account securityContext: runAsUser: 0 --- # Source: longhorn/templates/deployment-ui.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-ui namespace: longhorn-system spec: replicas: 2 selector: matchLabels: app: longhorn-ui template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui spec: serviceAccountName: longhorn-ui-service-account affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - longhorn-ui topologyKey: kubernetes.io/hostname weight: 1 containers: - name: oauth-proxy image: "" imagePullPolicy: IfNotPresent ports: - containerPort: 8443 name: public args: - --https-address=:8443 - --provider=openshift - --openshift-service-account=longhorn-ui-service-account - --upstream=http://localhost:8000 - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET - --openshift-sar={"namespace":"longhorn-system","group":"longhorn.io","resource":"setting","verb":"delete"} volumeMounts: - mountPath: /etc/tls/private name: longhorn-ui-tls - name: longhorn-ui image: docker.io/longhornio/longhorn-ui:master-head imagePullPolicy: IfNotPresent volumeMounts: - name: nginx-cache mountPath: /var/cache/nginx/ - name: nginx-config mountPath: /var/config/nginx/ - name: var-run mountPath: /var/run/ ports: - containerPort: 8000 name: http env: - name: LONGHORN_MANAGER_IP value: "http://longhorn-backend:9500" - name: LONGHORN_UI_PORT value: "8000" volumes: - name: longhorn-ui-tls secret: secretName: longhorn-ui-tls - emptyDir: {} name: nginx-cache - emptyDir: {} name: nginx-config - emptyDir: {} name: var-run priorityClassName: "longhorn-critical" --- # Source: longhorn/templates/validate-psp-install.yaml # --- # Source: longhorn/templates/deployment-ui.yaml # https://github.com/openshift/oauth-proxy/blob/master/contrib/sidecar.yaml # Create a proxy service account and ensure it will use the route "proxy" # Create a secure connection to the proxy via a route apiVersion: route.openshift.io/v1 kind: Route metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-ui namespace: longhorn-system spec: to: kind: Service name: longhorn-ui tls: termination: reencrypt ================================================ FILE: deploy/longhorn.yaml ================================================ --- # Builtin: "helm template" does not respect --create-namespace apiVersion: v1 kind: Namespace metadata: name: longhorn-system --- # Source: longhorn/templates/priorityclass.yaml apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: "longhorn-critical" labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" globalDefault: false preemptionPolicy: PreemptLowerPriority value: 1000000000 --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-service-account namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-ui-service-account namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev --- # Source: longhorn/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-support-bundle namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev --- # Source: longhorn/templates/default-resource.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-resource namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: default-resource.yaml: |- --- # Source: longhorn/templates/default-setting.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-setting namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: default-setting.yaml: |- priority-class: "longhorn-critical" disable-revision-counter: "{\"v1\":\"true\"}" --- # Source: longhorn/templates/storageclass.yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-storageclass namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev data: storageclass.yaml: | kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn annotations: storageclass.kubernetes.io/is-default-class: "true" provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: "Delete" volumeBindingMode: Immediate parameters: numberOfReplicas: "3" staleReplicaTimeout: "30" fromBackup: "" fsType: "ext4" dataLocality: "disabled" unmapMarkSnapChainRemoved: "ignored" disableRevisionCounter: "true" dataEngine: "v1" backupTargetName: "default" --- # Source: longhorn/templates/crds.yaml # Generated crds.yaml from github.com/longhorn/longhorn-manager/k8s/pkg/apis and the crds.yaml will be copied to longhorn/longhorn chart/templates and cannot be directly used by kubectl apply. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimagedatasources.longhorn.io spec: group: longhorn.io names: kind: BackingImageDataSource listKind: BackingImageDataSourceList plural: backingimagedatasources shortNames: - lhbids singular: backingimagedatasource scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID of the provisioned backing image file jsonPath: .spec.uuid name: UUID type: string - description: The current state of the pod used to provision the backing image file from source jsonPath: .status.currentState name: State type: string - description: The data source type jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size jsonPath: .status.size name: Size type: string - description: The node the backing image file will be prepared on jsonPath: .spec.nodeID name: Node type: string - description: The disk the backing image file will be prepared on jsonPath: .spec.diskUUID name: DiskUUID type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageDataSource is where Longhorn stores backing image data source object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source properties: checksum: type: string diskPath: type: string diskUUID: type: string fileTransferred: type: boolean nodeID: type: string parameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string uuid: type: string type: object status: description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source properties: checksum: type: string currentState: type: string ip: type: string message: type: string ownerID: type: string progress: type: integer runningParameters: additionalProperties: type: string nullable: true type: object size: format: int64 type: integer storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimagemanagers.longhorn.io spec: group: longhorn.io names: kind: BackingImageManager listKind: BackingImageManagerList plural: backingimagemanagers shortNames: - lhbim singular: backingimagemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The current state of the manager jsonPath: .status.currentState name: State type: string - description: The image the manager pod will use jsonPath: .spec.image name: Image type: string - description: The node the manager is on jsonPath: .spec.nodeID name: Node type: string - description: The disk the manager is responsible for jsonPath: .spec.diskUUID name: DiskUUID type: string - description: The disk path the manager is using jsonPath: .spec.diskPath name: DiskPath type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImageManager is where Longhorn stores backing image manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager properties: backingImages: additionalProperties: type: string type: object diskPath: type: string diskUUID: type: string image: type: string nodeID: type: string type: object status: description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImageFileMap: additionalProperties: properties: currentChecksum: type: string message: type: string name: type: string progress: type: integer realSize: format: int64 type: integer senderManagerAddress: type: string sendingReference: type: integer size: format: int64 type: integer state: type: string uuid: type: string virtualSize: format: int64 type: integer type: object nullable: true type: object currentState: type: string ip: type: string ownerID: type: string storageIP: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backingimages.longhorn.io spec: group: longhorn.io names: kind: BackingImage listKind: BackingImageList plural: backingimages shortNames: - lhbi singular: backingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The system generated UUID jsonPath: .status.uuid name: UUID type: string - description: The source of the backing image file data jsonPath: .spec.sourceType name: SourceType type: string - description: The backing image file size in each disk jsonPath: .status.size name: Size type: string - description: The virtual size of the image (may be larger than file size) jsonPath: .status.virtualSize name: VirtualSize type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: BackingImage is where Longhorn stores backing image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackingImageSpec defines the desired state of the Longhorn backing image properties: checksum: type: string dataEngine: default: v1 enum: - v1 - v2 type: string diskFileSpecMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string evictionRequested: type: boolean type: object type: object diskSelector: items: type: string type: array disks: additionalProperties: type: string description: Deprecated. We are now using DiskFileSpecMap to assign different spec to the file on different disks. type: object minNumberOfCopies: type: integer nodeSelector: items: type: string type: array secret: type: string secretNamespace: type: string sourceParameters: additionalProperties: type: string type: object sourceType: enum: - download - upload - export-from-volume - restore - clone type: string type: object status: description: BackingImageStatus defines the observed state of the Longhorn backing image status properties: checksum: type: string diskFileStatusMap: additionalProperties: properties: dataEngine: enum: - v1 - v2 type: string lastStateTransitionTime: type: string message: type: string progress: type: integer state: type: string type: object nullable: true type: object diskLastRefAtMap: additionalProperties: type: string nullable: true type: object ownerID: type: string realSize: description: Real size of image in bytes, which may be smaller than the size when the file is a sparse file. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer size: format: int64 type: integer uuid: type: string v2FirstCopyDisk: type: string v2FirstCopyStatus: description: It is pending -> in-progress -> ready/failed type: string virtualSize: description: Virtual size of image in bytes, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading) format: int64 type: integer type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backupbackingimages.longhorn.io spec: group: longhorn.io names: kind: BackupBackingImage listKind: BackupBackingImageList plural: backupbackingimages shortNames: - lhbbi singular: backupbackingimage scope: Namespaced versions: - additionalPrinterColumns: - description: The backing image name jsonPath: .status.backingImage name: BackingImage type: string - description: The backing image size jsonPath: .status.size name: Size type: string - description: The backing image backup upload finished time jsonPath: .status.backupCreatedAt name: BackupCreatedAt type: string - description: The backing image backup state jsonPath: .status.state name: State type: string - description: The last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupBackingImage is where Longhorn stores backing image backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupTargetName: description: The backup target name. nullable: true type: string labels: additionalProperties: type: string description: The labels of backing image backup. type: object syncRequestedAt: description: The time to request run sync the remote backing image backup. format: date-time nullable: true type: string userCreated: description: Is this CR created by user through API or UI. type: boolean required: - backingImage - userCreated type: object status: description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup properties: backingImage: description: The backing image name. type: string backupCreatedAt: description: The backing image backup upload finished time. type: string checksum: description: The checksum of the backing image. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the backing image backup. type: string labels: additionalProperties: type: string description: The labels of backing image backup. nullable: true type: object lastSyncedAt: description: The last time that the backing image backup was synced with the remote backup target. format: date-time nullable: true type: string managerAddress: description: The address of the backing image manager that runs backing image backup. type: string messages: additionalProperties: type: string description: The error messages when listing or inspecting backing image backup. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this CR. type: string progress: description: The backing image backup progress. type: integer secret: description: Record the secret if this backup backing image is encrypted type: string secretNamespace: description: Record the secret namespace if this backup backing image is encrypted type: string size: description: The backing image size. format: int64 type: integer state: description: |- The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The backing image backup URL. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backups.longhorn.io spec: group: longhorn.io names: kind: Backup listKind: BackupList plural: backups shortNames: - lhb singular: backup scope: Namespaced versions: - additionalPrinterColumns: - description: The snapshot name jsonPath: .status.snapshotName name: SnapshotName type: string - description: The snapshot size jsonPath: .status.size name: SnapshotSize type: string - description: The snapshot creation time jsonPath: .status.snapshotCreatedAt name: SnapshotCreatedAt type: string - description: The backup target name jsonPath: .status.backupTargetName name: BackupTarget type: string - description: The backup state jsonPath: .status.state name: State type: string - description: The backup last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: Backup is where Longhorn stores backup object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupSpec defines the desired state of the Longhorn backup properties: backupBlockSize: description: The backup block size. 0 means the legacy default size 2MiB, and -1 indicate the block size is invalid. enum: - "-1" - "2097152" - "16777216" format: int64 type: string backupMode: description: |- The backup mode of this backup. Can be "full" or "incremental" enum: - full - incremental type: string labels: additionalProperties: type: string description: The labels of snapshot backup. type: object snapshotName: description: The snapshot name. type: string syncRequestedAt: description: The time to request run sync the remote backup. format: date-time nullable: true type: string type: object status: description: BackupStatus defines the observed state of the Longhorn backup properties: backupCreatedAt: description: The snapshot backup upload finished time. type: string backupTargetName: description: The backup target name. type: string compressionMethod: description: Compression method type: string error: description: The error message when taking the snapshot backup. type: string labels: additionalProperties: type: string description: The labels of snapshot backup. nullable: true type: object lastSyncedAt: description: The last time that the backup was synced with the remote backup target. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when calling longhorn engine on listing or inspecting backups. nullable: true type: object newlyUploadDataSize: description: Size in bytes of newly uploaded data type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup CR. type: string progress: description: The snapshot backup progress. type: integer reUploadedDataSize: description: Size in bytes of reuploaded data type: string replicaAddress: description: The address of the replica that runs snapshot backup. type: string size: description: The snapshot size. type: string snapshotCreatedAt: description: The snapshot creation time. type: string snapshotName: description: The snapshot name. type: string state: description: |- The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". type: string url: description: The snapshot backup URL. type: string volumeBackingImageName: description: The volume's backing image name. type: string volumeCreated: description: The volume creation time. type: string volumeName: description: The volume name. type: string volumeSize: description: The volume size. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backuptargets.longhorn.io spec: group: longhorn.io names: kind: BackupTarget listKind: BackupTargetList plural: backuptargets shortNames: - lhbt singular: backuptarget scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target URL jsonPath: .spec.backupTargetURL name: URL type: string - description: The backup target credential secret jsonPath: .spec.credentialSecret name: Credential type: string - description: The backup target poll interval jsonPath: .spec.pollInterval name: LastBackupAt type: string - description: Indicate whether the backup target is available or not jsonPath: .status.available name: Available type: boolean - description: The backup target last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupTarget is where Longhorn stores backup target object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupTargetSpec defines the desired state of the Longhorn backup target properties: backupTargetURL: description: The backup target URL. type: string credentialSecret: description: The backup target credential secret. type: string pollInterval: description: The interval that the cluster needs to run sync with the backup target. type: string syncRequestedAt: description: The time to request run sync the remote backup target. format: date-time nullable: true type: string type: object status: description: BackupTargetStatus defines the observed state of the Longhorn backup target properties: available: description: Available indicates if the remote backup target is available or not. type: boolean conditions: description: Records the reason on why the backup target is unavailable. items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array lastSyncedAt: description: The last time that the controller synced with the remote backup target. format: date-time nullable: true type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup target CR. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: backupvolumes.longhorn.io spec: group: longhorn.io names: kind: BackupVolume listKind: BackupVolumeList plural: backupvolumes shortNames: - lhbv singular: backupvolume scope: Namespaced versions: - additionalPrinterColumns: - description: The backup target name jsonPath: .spec.backupTargetName name: BackupTarget type: string - description: The backup volume creation time jsonPath: .status.createdAt name: CreatedAt type: string - description: The backup volume last backup name jsonPath: .status.lastBackupName name: LastBackupName type: string - description: The backup volume last backup time jsonPath: .status.lastBackupAt name: LastBackupAt type: string - description: The backup volume last synced time jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: BackupVolume is where Longhorn stores backup volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: BackupVolumeSpec defines the desired state of the Longhorn backup volume properties: backupTargetName: description: The backup target name that the backup volume was synced. nullable: true type: string syncRequestedAt: description: The time to request run sync the remote backup volume. format: date-time nullable: true type: string volumeName: description: The volume name that the backup volume was used to backup. type: string type: object status: description: BackupVolumeStatus defines the observed state of the Longhorn backup volume properties: backingImageChecksum: description: the backing image checksum. type: string backingImageName: description: The backing image name. type: string createdAt: description: The backup volume creation time. type: string dataStored: description: The backup volume block count. type: string labels: additionalProperties: type: string description: The backup volume labels. nullable: true type: object lastBackupAt: description: The latest volume backup time. type: string lastBackupName: description: The latest volume backup name. type: string lastModificationTime: description: The backup volume config last modification time. format: date-time nullable: true type: string lastSyncedAt: description: The last time that the backup volume was synced into the cluster. format: date-time nullable: true type: string messages: additionalProperties: type: string description: The error messages when call longhorn engine on list or inspect backup volumes. nullable: true type: object ownerID: description: The node ID on which the controller is responsible to reconcile this backup volume CR. type: string size: description: The backup volume size. type: string storageClassName: description: the storage class name of pv/pvc binding with the volume. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: engineimages.longhorn.io spec: group: longhorn.io names: kind: EngineImage listKind: EngineImageList plural: engineimages shortNames: - lhei singular: engineimage scope: Namespaced versions: - additionalPrinterColumns: - description: Compatibility of the engine image jsonPath: .status.incompatible name: Incompatible type: boolean - description: State of the engine image jsonPath: .status.state name: State type: string - description: The Longhorn engine image jsonPath: .spec.image name: Image type: string - description: Number of resources using the engine image jsonPath: .status.refCount name: RefCount type: integer - description: The build date of the engine image jsonPath: .status.buildDate name: BuildDate type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: EngineImage is where Longhorn stores engine image object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineImageSpec defines the desired state of the Longhorn engine image properties: image: minLength: 1 type: string required: - image type: object status: description: EngineImageStatus defines the observed state of the Longhorn engine image properties: buildDate: type: string cliAPIMinVersion: type: integer cliAPIVersion: type: integer conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array controllerAPIMinVersion: type: integer controllerAPIVersion: type: integer dataFormatMinVersion: type: integer dataFormatVersion: type: integer gitCommit: type: string incompatible: type: boolean noRefSince: type: string nodeDeploymentMap: additionalProperties: type: boolean nullable: true type: object ownerID: type: string refCount: type: integer state: type: string version: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: engines.longhorn.io spec: group: longhorn.io names: kind: Engine listKind: EngineList plural: engines shortNames: - lhe singular: engine scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the engine jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the engine jsonPath: .status.currentState name: State type: string - description: The node that the engine is on jsonPath: .spec.nodeID name: Node type: string - description: The instance manager of the engine jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the engine jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Engine is where Longhorn stores engine object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: EngineSpec defines the desired state of the Longhorn engine properties: active: type: boolean backupVolume: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string disableFrontend: type: boolean frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string logRequested: type: boolean nodeID: type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. maximum: 5 minimum: 0 type: integer replicaAddressMap: additionalProperties: type: string type: object requestedBackupRestore: type: string requestedDataSource: type: string revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemovedEnabled: type: boolean upgradedReplicaAddressMap: additionalProperties: type: string type: object volumeName: type: string volumeSize: format: int64 type: string type: object status: description: EngineStatus defines the observed state of the Longhorn engine properties: backupStatus: additionalProperties: properties: backupURL: type: string error: type: string progress: type: integer replicaAddress: type: string snapshotName: type: string state: type: string type: object nullable: true type: object cloneStatus: additionalProperties: properties: error: type: string fromReplicaAddress: type: string isCloning: type: boolean progress: type: integer snapshotName: type: string state: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentReplicaAddressMap: additionalProperties: type: string nullable: true type: object currentSize: format: int64 type: string currentState: type: string endpoint: type: string instanceManagerName: type: string ip: type: string isExpanding: type: boolean lastExpansionError: type: string lastExpansionFailedAt: type: string lastRestoredBackup: type: string logFetched: type: boolean ownerID: type: string port: type: integer purgeStatus: additionalProperties: properties: error: type: string isPurging: type: boolean progress: type: integer state: type: string type: object nullable: true type: object rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. It is determined by the global setting or the volume spec field with the same name. minimum: 0 type: integer rebuildStatus: additionalProperties: properties: appliedRebuildingMBps: format: int64 type: integer error: type: string fromReplicaAddress: description: Deprecated. We are now using FromReplicaAddressList to list all source replicas. type: string fromReplicaAddressList: items: type: string type: array isRebuilding: type: boolean progress: type: integer state: type: string type: object nullable: true type: object replicaModeMap: additionalProperties: type: string nullable: true type: object replicaTransitionTimeMap: additionalProperties: type: string description: |- ReplicaTransitionTimeMap records the time a replica in ReplicaModeMap transitions from one mode to another (or from not being in the ReplicaModeMap to being in it). This information is sometimes required by other controllers (e.g. the volume controller uses it to determine the correct value for replica.Spec.lastHealthyAt). type: object restoreStatus: additionalProperties: properties: backupURL: type: string currentRestoringBackup: type: string error: type: string filename: type: string isRestoring: type: boolean lastRestored: type: string progress: type: integer state: type: string type: object nullable: true type: object salvageExecuted: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string snapshots: additionalProperties: properties: children: additionalProperties: type: boolean nullable: true type: object created: type: string labels: additionalProperties: type: string nullable: true type: object name: type: string parent: type: string removed: type: boolean size: type: string usercreated: type: boolean type: object nullable: true type: object snapshotsError: type: string started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer unmapMarkSnapChainRemovedEnabled: type: boolean uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: instancemanagers.longhorn.io spec: group: longhorn.io names: kind: InstanceManager listKind: InstanceManagerList plural: instancemanagers shortNames: - lhim singular: instancemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the instance manager jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the instance manager jsonPath: .status.currentState name: State type: string - description: The type of the instance manager (engine or replica) jsonPath: .spec.type name: Type type: string - description: The node that the instance manager is running on jsonPath: .spec.nodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: InstanceManager is where Longhorn stores instance manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: InstanceManagerSpec defines the desired state of the Longhorn instance manager properties: dataEngine: type: string dataEngineSpec: properties: v2: properties: cpuMask: type: string type: object type: object image: type: string nodeID: type: string type: enum: - aio - engine - replica type: string type: object status: description: InstanceManagerStatus defines the observed state of the Longhorn instance manager properties: apiMinVersion: type: integer apiVersion: type: integer backingImages: additionalProperties: properties: currentChecksum: type: string diskUUID: type: string message: type: string name: type: string progress: type: integer size: format: int64 type: integer state: type: string uuid: type: string type: object nullable: true type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentState: type: string dataEngineStatus: properties: v2: properties: cpuMask: type: string interruptModeEnabled: description: |- InterruptModeEnabled indicates whether the V2 data engine is running in interrupt mode (true) or polling mode (false). Set by Longhorn manager; read-only to users. enum: - "" - "true" - "false" type: string type: object type: object instanceEngines: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object instanceReplicas: additionalProperties: properties: spec: properties: dataEngine: type: string name: type: string type: object status: properties: conditions: additionalProperties: type: boolean nullable: true type: object endpoint: type: string errorMsg: type: string listen: type: string portEnd: format: int32 type: integer portStart: format: int32 type: integer resourceVersion: format: int64 type: integer state: type: string targetPortEnd: format: int32 type: integer targetPortStart: format: int32 type: integer type: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object nullable: true type: object ip: type: string ownerID: type: string proxyApiMinVersion: type: integer proxyApiVersion: type: integer type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: nodes.longhorn.io spec: group: longhorn.io names: kind: Node listKind: NodeList plural: nodes shortNames: - lhn singular: node scope: Namespaced versions: - additionalPrinterColumns: - description: Indicate whether the node is ready jsonPath: .status.conditions[?(@.type=='Ready')].status name: Ready type: string - description: Indicate whether the user disabled/enabled replica scheduling for the node jsonPath: .spec.allowScheduling name: AllowScheduling type: boolean - description: Indicate whether Longhorn can schedule replicas on the node jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Schedulable type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Node is where Longhorn stores Longhorn node object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: NodeSpec defines the desired state of the Longhorn node properties: allowScheduling: type: boolean disks: additionalProperties: properties: allowScheduling: type: boolean diskDriver: enum: - "" - auto - aio - nvme type: string diskType: enum: - filesystem - block type: string evictionRequested: type: boolean path: type: string storageReserved: format: int64 type: integer tags: items: type: string type: array type: object type: object evictionRequested: type: boolean instanceManagerCPURequest: type: integer name: type: string tags: items: type: string type: array type: object status: description: NodeStatus defines the observed state of the Longhorn node properties: autoEvicting: type: boolean conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskStatus: additionalProperties: properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array diskDriver: type: string diskName: type: string diskPath: type: string diskType: type: string diskUUID: type: string filesystemType: type: string healthData: additionalProperties: properties: attributes: items: properties: id: type: integer name: type: string rawString: type: string rawValue: format: int64 type: integer threshold: type: integer value: type: integer whenFailed: type: string worst: type: integer type: object type: array capacity: format: int64 type: integer diskName: type: string diskType: type: string firmwareVersion: type: string healthStatus: enum: - FAILED - PASSED - UNKNOWN - WARNING type: string modelName: type: string serialNumber: type: string source: enum: - SMART - SPDK type: string temperature: type: integer type: object type: object healthDataLastCollectedAt: format: date-time type: string instanceManagerName: type: string scheduledBackingImage: additionalProperties: format: int64 type: integer nullable: true type: object scheduledReplica: additionalProperties: format: int64 type: integer nullable: true type: object storageAvailable: format: int64 type: integer storageMaximum: format: int64 type: integer storageScheduled: format: int64 type: integer type: object nullable: true type: object region: type: string snapshotCheckStatus: properties: lastPeriodicCheckedAt: format: date-time type: string type: object zone: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: orphans.longhorn.io spec: group: longhorn.io names: kind: Orphan listKind: OrphanList plural: orphans shortNames: - lho singular: orphan scope: Namespaced versions: - additionalPrinterColumns: - description: The type of the orphan jsonPath: .spec.orphanType name: Type type: string - description: The node that the orphan is on jsonPath: .spec.nodeID name: Node type: string name: v1beta2 schema: openAPIV3Schema: description: Orphan is where Longhorn stores orphan object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: OrphanSpec defines the desired state of the Longhorn orphaned data properties: dataEngine: description: |- The type of data engine for instance orphan. Can be "v1", "v2". enum: - v1 - v2 type: string nodeID: description: The node ID on which the controller is responsible to reconcile this orphan CR. type: string orphanType: description: |- The type of the orphaned data. Can be "replica". type: string parameters: additionalProperties: type: string description: The parameters of the orphaned data type: object type: object status: description: OrphanStatus defines the observed state of the Longhorn orphaned data properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: recurringjobs.longhorn.io spec: group: longhorn.io names: kind: RecurringJob listKind: RecurringJobList plural: recurringjobs shortNames: - lhrj singular: recurringjob scope: Namespaced versions: - additionalPrinterColumns: - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume jsonPath: .spec.groups name: Groups type: string - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup" jsonPath: .spec.task name: Task type: string - description: The cron expression represents recurring job scheduling jsonPath: .spec.cron name: Cron type: string - description: The number of snapshots/backups to keep for the volume jsonPath: .spec.retain name: Retain type: integer - description: The concurrent job to run by each cron job jsonPath: .spec.concurrency name: Concurrency type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date - description: Specify the labels jsonPath: .spec.labels name: Labels type: string name: v1beta2 schema: openAPIV3Schema: description: RecurringJob is where Longhorn stores recurring job object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: RecurringJobSpec defines the desired state of the Longhorn recurring job properties: concurrency: description: The concurrency of taking the snapshot/backup. type: integer cron: description: The cron setting. type: string groups: description: The recurring job group. items: type: string type: array labels: additionalProperties: type: string description: The label of the snapshot/backup. type: object name: description: The recurring job name. type: string parameters: additionalProperties: type: string description: |- The parameters of the snapshot/backup. Support parameters: "full-backup-interval", "volume-backup-policy". type: object retain: description: The retain count of the snapshot/backup. type: integer task: description: |- The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create", "filesystem-trim" or "system-backup". enum: - snapshot - snapshot-force-create - snapshot-cleanup - snapshot-delete - backup - backup-force-create - filesystem-trim - system-backup type: string type: object status: description: RecurringJobStatus defines the observed state of the Longhorn recurring job properties: executionCount: description: The number of jobs that have been triggered. type: integer ownerID: description: The owner ID which is responsible to reconcile this recurring job CR. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: replicas.longhorn.io spec: group: longhorn.io names: kind: Replica listKind: ReplicaList plural: replicas shortNames: - lhr singular: replica scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the replica jsonPath: .spec.dataEngine name: Data Engine type: string - description: The current state of the replica jsonPath: .status.currentState name: State type: string - description: The node that the replica is on jsonPath: .spec.nodeID name: Node type: string - description: The disk that the replica is on jsonPath: .spec.diskID name: Disk type: string - description: The instance manager of the replica jsonPath: .status.instanceManagerName name: InstanceManager type: string - description: The current image of the replica jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Replica is where Longhorn stores replica object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ReplicaSpec defines the desired state of the Longhorn replica properties: active: type: boolean backingImage: type: string dataDirectoryName: type: string dataEngine: enum: - v1 - v2 type: string desireState: type: string diskID: type: string diskPath: type: string engineName: type: string evictionRequested: type: boolean failedAt: description: |- FailedAt is set when a running replica fails or when a running engine is unable to use a replica for any reason. FailedAt indicates the time the failure occurred. When FailedAt is set, a replica is likely to have useful (though possibly stale) data. A replica with FailedAt set must be rebuilt from a non-failed replica (or it can be used in a salvage if all replicas are failed). FailedAt is cleared before a rebuild or salvage. FailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string hardNodeAffinity: type: string healthyAt: description: |- HealthyAt is set the first time a replica becomes read/write in an engine after creation or rebuild. HealthyAt indicates the time the last successful rebuild occurred. When HealthyAt is set, a replica is likely to have useful (though possibly stale) data. HealthyAt is cleared before a rebuild. HealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string image: type: string lastFailedAt: description: |- LastFailedAt is always set at the same time as FailedAt. Unlike FailedAt, LastFailedAt is never cleared. LastFailedAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastFailedAt may already be healthy and in use again. However, because it is never cleared, it can be compared to LastHealthyAt to help prevent dangerous replica deletion in some corner cases. LastFailedAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string lastHealthyAt: description: |- LastHealthyAt is set every time a replica becomes read/write in an engine. Unlike HealthyAt, LastHealthyAt is never cleared. LastHealthyAt is not a reliable indicator of the state of a replica's data. For example, a replica with LastHealthyAt set may be in the middle of a rebuild. However, because it is never cleared, it can be compared to LastFailedAt to help prevent dangerous replica deletion in some corner cases. LastHealthyAt may be later than the corresponding entry in an engine's replicaTransitionTimeMap because it is set when the volume controller acknowledges the change. type: string logRequested: type: boolean migrationEngineName: description: |- MigrationEngineName is indicating the migrating engine which current connected to this replica. This is only used for live migration of v2 data engine type: string nodeID: type: string rebuildRetryCount: type: integer revisionCounterDisabled: type: boolean salvageRequested: type: boolean snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string unmapMarkDiskChainRemovedEnabled: type: boolean volumeName: type: string volumeSize: format: int64 type: string type: object status: description: ReplicaStatus defines the observed state of the Longhorn replica properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentState: type: string instanceManagerName: type: string ip: type: string logFetched: type: boolean ownerID: type: string port: type: integer salvageExecuted: type: boolean started: type: boolean starting: type: boolean storageIP: type: string ublkID: format: int32 type: integer uuid: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: settings.longhorn.io spec: group: longhorn.io names: kind: Setting listKind: SettingList plural: settings shortNames: - lhs singular: setting scope: Namespaced versions: - additionalPrinterColumns: - description: The value of the setting jsonPath: .value name: Value type: string - description: The setting is applied jsonPath: .status.applied name: Applied type: boolean - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Setting is where Longhorn stores setting object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object status: description: The status of the setting. properties: applied: description: The setting is applied. type: boolean required: - applied type: object value: description: |- The value of the setting. - It can be a non-JSON formatted string that is applied to all the applicable data engines listed in the setting definition. - It can be a JSON formatted string that contains values for applicable data engines listed in the setting definition's Default. type: string required: - value type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: sharemanagers.longhorn.io spec: group: longhorn.io names: kind: ShareManager listKind: ShareManagerList plural: sharemanagers shortNames: - lhsm singular: sharemanager scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the share manager jsonPath: .status.state name: State type: string - description: The node that the share manager is owned by jsonPath: .status.ownerID name: Node type: string - description: The current image of the share manager jsonPath: .status.currentImage name: Image type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: ShareManager is where Longhorn stores share manager object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ShareManagerSpec defines the desired state of the Longhorn share manager properties: image: description: Share manager image used for creating a share manager pod type: string type: object status: description: ShareManagerStatus defines the observed state of the Longhorn share manager properties: currentImage: description: The image currently used by the share manager pod type: string endpoint: description: NFS endpoint that can access the mounted filesystem of the volume type: string ownerID: description: The node ID on which the controller is responsible to reconcile this share manager resource type: string state: description: The state of the share manager resource type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: snapshots.longhorn.io spec: group: longhorn.io names: kind: Snapshot listKind: SnapshotList plural: snapshots shortNames: - lhsnap singular: snapshot scope: Namespaced versions: - additionalPrinterColumns: - description: The volume that this snapshot belongs to jsonPath: .spec.volume name: Volume type: string - description: Timestamp when the point-in-time snapshot was taken jsonPath: .status.creationTime name: CreationTime type: string - description: Indicates if the snapshot is ready to be used to restore/backup a volume jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Represents the minimum size of volume required to rehydrate from this snapshot jsonPath: .status.restoreSize name: RestoreSize type: string - description: The actual size of the snapshot jsonPath: .status.size name: Size type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Snapshot is the Schema for the snapshots API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SnapshotSpec defines the desired state of Longhorn Snapshot properties: createSnapshot: description: require creating a new snapshot type: boolean labels: additionalProperties: type: string description: The labels of snapshot nullable: true type: object volume: description: |- the volume that this snapshot belongs to. This field is immutable after creation. type: string required: - volume type: object status: description: SnapshotStatus defines the observed state of Longhorn Snapshot properties: checksum: type: string checksumCalculatedAt: description: |- ChecksumCalculatedAt is the RFC3339 timestamp indicating when the checksum for this snapshot was last calculated or updated. type: string children: additionalProperties: type: boolean nullable: true type: object creationTime: type: string error: type: string labels: additionalProperties: type: string nullable: true type: object markRemoved: type: boolean ownerID: type: string parent: type: string readyToUse: type: boolean restoreSize: format: int64 type: integer size: format: int64 type: integer userCreated: type: boolean type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: supportbundles.longhorn.io spec: group: longhorn.io names: kind: SupportBundle listKind: SupportBundleList plural: supportbundles shortNames: - lhbundle singular: supportbundle scope: Namespaced versions: - additionalPrinterColumns: - description: The state of the support bundle jsonPath: .status.state name: State type: string - description: The issue URL jsonPath: .spec.issueURL name: Issue type: string - description: A brief description of the issue jsonPath: .spec.description name: Description type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SupportBundle is where Longhorn stores support bundle object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle properties: description: description: A brief description of the issue type: string issueURL: description: The issue URL nullable: true type: string nodeID: description: The preferred responsible controller node ID. type: string required: - description type: object status: description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object type: array filename: type: string filesize: format: int64 type: integer image: description: The support bundle manager image type: string managerIP: description: The support bundle manager IP type: string ownerID: description: The current responsible controller node ID type: string progress: type: integer state: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: systembackups.longhorn.io spec: group: longhorn.io names: kind: SystemBackup listKind: SystemBackupList plural: systembackups shortNames: - lhsb singular: systembackup scope: Namespaced versions: - additionalPrinterColumns: - description: The system backup Longhorn version jsonPath: .status.version name: Version type: string - description: The system backup state jsonPath: .status.state name: State type: string - description: The system backup creation time jsonPath: .status.createdAt name: Created type: string - description: The last time that the system backup was synced into the cluster jsonPath: .status.lastSyncedAt name: LastSyncedAt type: string name: v1beta2 schema: openAPIV3Schema: description: SystemBackup is where Longhorn stores system backup object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup properties: volumeBackupPolicy: description: |- The create volume backup policy Can be "if-not-present", "always" or "disabled" nullable: true type: string type: object status: description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array createdAt: description: The system backup creation time. format: date-time type: string gitCommit: description: The saved Longhorn manager git commit. nullable: true type: string lastSyncedAt: description: The last time that the system backup was synced into the cluster. format: date-time nullable: true type: string managerImage: description: The saved manager image. type: string ownerID: description: The node ID of the responsible controller to reconcile this SystemBackup. type: string state: description: The system backup state. type: string version: description: The saved Longhorn version. nullable: true type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: systemrestores.longhorn.io spec: group: longhorn.io names: kind: SystemRestore listKind: SystemRestoreList plural: systemrestores shortNames: - lhsr singular: systemrestore scope: Namespaced versions: - additionalPrinterColumns: - description: The system restore state jsonPath: .status.state name: State type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: SystemRestore is where Longhorn stores system restore object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore properties: systemBackup: description: The system backup name in the object store. type: string required: - systemBackup type: object status: description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore properties: conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array ownerID: description: The node ID of the responsible controller to reconcile this SystemRestore. type: string sourceURL: description: The source system backup URL. type: string state: description: The system restore state. type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: volumeattachments.longhorn.io spec: group: longhorn.io names: kind: VolumeAttachment listKind: VolumeAttachmentList plural: volumeattachments shortNames: - lhva singular: volumeattachment scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: VolumeAttachment stores attachment information of a Longhorn volume properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment properties: attachmentTickets: additionalProperties: properties: generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string nodeID: description: The node that this attachment is requesting type: string parameters: additionalProperties: type: string description: Optional additional parameter for this attachment type: object type: type: string type: object type: object volume: description: The name of Longhorn volume of this VolumeAttachment type: string required: - volume type: object status: description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment properties: attachmentTicketStatuses: additionalProperties: properties: conditions: description: Record any error when trying to fulfill this attachment items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array generation: description: |- A sequence number representing a specific generation of the desired state. Populated by the system. Read-only. format: int64 type: integer id: description: The unique ID of this attachment. Used to differentiate different attachments of the same volume. type: string satisfied: description: Indicate whether this attachment ticket has been satisfied type: boolean required: - conditions - satisfied type: object type: object type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev longhorn-manager: "" name: volumes.longhorn.io spec: group: longhorn.io names: kind: Volume listKind: VolumeList plural: volumes shortNames: - lhv singular: volume scope: Namespaced versions: - additionalPrinterColumns: - description: The data engine of the volume jsonPath: .spec.dataEngine name: Data Engine type: string - description: The state of the volume jsonPath: .status.state name: State type: string - description: The robustness of the volume jsonPath: .status.robustness name: Robustness type: string - description: The scheduled condition of the volume jsonPath: .status.conditions[?(@.type=='Schedulable')].status name: Scheduled type: string - description: The size of the volume jsonPath: .spec.size name: Size type: string - description: The node that the volume is currently attaching to jsonPath: .status.currentNodeID name: Node type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta2 schema: openAPIV3Schema: description: Volume is where Longhorn stores volume object. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: VolumeSpec defines the desired state of the Longhorn volume properties: Standby: type: boolean accessMode: enum: - rwo - rwop - rwx type: string backingImage: type: string x-kubernetes-validations: - message: BackingImage is immutable rule: self == oldSelf backupBlockSize: description: BackupBlockSize indicate the block size to create backups. The block size is immutable. enum: - "2097152" - "16777216" format: int64 type: string backupCompressionMethod: enum: - none - lz4 - gzip type: string backupTargetName: description: The backup target name that the volume will be backed up to or is synced. type: string cloneMode: enum: - "" - full-copy - linked-clone type: string dataEngine: enum: - v1 - v2 type: string dataLocality: enum: - disabled - best-effort - strict-local type: string dataSource: type: string disableFrontend: type: boolean diskSelector: items: type: string type: array encrypted: type: boolean x-kubernetes-validations: - message: Encrypted is immutable rule: self == oldSelf freezeFilesystemForSnapshot: description: Setting that freezes the filesystem on the root partition before a snapshot is created. enum: - ignored - enabled - disabled type: string fromBackup: type: string frontend: enum: - blockdev - iscsi - nvmf - ublk - "" type: string image: type: string lastAttachedBy: type: string migratable: type: boolean migrationNodeID: type: string nodeID: type: string nodeSelector: items: type: string type: array numberOfReplicas: type: integer offlineRebuilding: description: |- Specifies whether Longhorn should rebuild replicas while the detached volume is degraded. - ignored: Use the global setting for offline replica rebuilding. - enabled: Enable offline rebuilding for this volume, regardless of the global setting. - disabled: Disable offline rebuilding for this volume, regardless of the global setting enum: - ignored - disabled - enabled type: string rebuildConcurrentSyncLimit: description: |- RebuildConcurrentSyncLimit controls the maximum number of file synchronization operations that can run concurrently during a single replica rebuild. When set to 0, it means following the global setting. maximum: 5 minimum: 0 type: integer replicaAutoBalance: enum: - ignored - disabled - least-effort - best-effort type: string replicaDiskSoftAntiAffinity: description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. enum: - ignored - enabled - disabled type: string replicaRebuildingBandwidthLimit: description: ReplicaRebuildingBandwidthLimit controls the maximum write bandwidth (in megabytes per second) allowed on the destination replica during the rebuilding process. Set this value to 0 to disable bandwidth limiting. format: int64 minimum: 0 type: integer replicaSoftAntiAffinity: description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node. enum: - ignored - enabled - disabled type: string replicaZoneSoftAntiAffinity: description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone. enum: - ignored - enabled - disabled type: string restoreVolumeRecurringJob: enum: - ignored - enabled - disabled type: string revisionCounterDisabled: type: boolean size: format: int64 type: string snapshotDataIntegrity: enum: - ignored - disabled - enabled - fast-check type: string snapshotHashingRequestedAt: description: |- SnapshotHashingRequestedAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when an on-demand snapshot checksum calculation is requested. When this value is set and is later than LastOnDemandSnapshotHashingCompleteAt, the system will calculate checksums for all user snapshots. If SnapshotHashingRequestedAt differs from LastOnDemandSnapshotHashingCompleteAt, it indicates that a hashing request is still in progress, and a new request will be rejected. type: string snapshotMaxCount: type: integer snapshotMaxSize: format: int64 type: string staleReplicaTimeout: type: integer ublkNumberOfQueue: description: ublkNumberOfQueue controls the number of queues for ublk frontend. type: integer ublkQueueDepth: description: ublkQueueDepth controls the depth of each queue for ublk frontend. type: integer unmapMarkSnapChainRemoved: enum: - ignored - disabled - enabled type: string type: object status: description: VolumeStatus defines the observed state of the Longhorn volume properties: actualSize: format: int64 type: integer cloneStatus: properties: attemptCount: type: integer nextAllowedAttemptAt: type: string snapshot: type: string sourceVolume: type: string state: type: string type: object conditions: items: properties: lastProbeTime: description: Last time we probed the condition. type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string status: description: |- Status is the status of the condition. Can be True, False, Unknown. type: string type: description: Type is the type of the condition. type: string type: object nullable: true type: array currentImage: type: string currentMigrationNodeID: description: the node that this volume is currently migrating to type: string currentNodeID: type: string expansionRequired: type: boolean frontendDisabled: type: boolean isStandby: type: boolean kubernetesStatus: properties: lastPVCRefAt: type: string lastPodRefAt: type: string namespace: description: determine if PVC/Namespace is history or not type: string pvName: type: string pvStatus: type: string pvcName: type: string workloadsStatus: description: determine if Pod/Workload is history or not items: properties: podName: type: string podStatus: type: string workloadName: type: string workloadType: type: string type: object nullable: true type: array type: object lastBackup: type: string lastBackupAt: type: string lastDegradedAt: type: string lastOnDemandSnapshotHashingCompleteAt: description: |- LastOnDemandSnapshotHashingCompleteAt is the RFC3339 timestamp (e.g., "2026-03-16T10:30:00Z") when the most recent on-demand snapshot checksum calculation completed. When this value matches SnapshotHashingRequestedAt, the requested on-demand checksum calculation is considered complete. type: string ownerID: type: string remountRequestedAt: type: string restoreInitiated: type: boolean restoreRequired: type: boolean robustness: type: string shareEndpoint: type: string shareState: type: string state: type: string type: object type: object served: true storage: true subresources: status: {} --- # Source: longhorn/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-role labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev rules: - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - "*" - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "delete", "deletecollection"] - apiGroups: [""] resources: ["secrets", "services", "endpoints", "configmaps", "serviceaccounts", "pods/log"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status", "nodes"] verbs: ["*"] - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "statefulsets", "deployments", "replicasets"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "list", "watch"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["watch", "list"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "volumeattributesclasses", "csinodes", "csidrivers", "csistoragecapacities"] verbs: ["*"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] verbs: ["*"] - apiGroups: ["longhorn.io"] resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", "settings/status", "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status", "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch"] - apiGroups: ["metrics.k8s.io"] resources: ["pods", "nodes"] verbs: ["get", "list"] - apiGroups: ["apiregistration.k8s.io"] resources: ["apiservices"] verbs: ["list", "watch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] verbs: ["get", "list", "create", "patch", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterrolebindings", "clusterroles"] verbs: ["*"] --- # Source: longhorn/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-bind labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system --- # Source: longhorn/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-support-bundle labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: longhorn-support-bundle namespace: longhorn-system --- # Source: longhorn/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: longhorn namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev rules: - apiGroups: [""] resources: ["pods", "pods/log", "events", "secrets", "services", "endpoints", "configmaps", "serviceaccounts", "persistentvolumeclaims", "persistentvolumeclaims/status"] verbs: ["*"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "statefulsets", "replicasets"] verbs: ["*"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["*"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["*"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["*"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["*"] --- # Source: longhorn/templates/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: longhorn namespace: longhorn-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: longhorn subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system --- # Source: longhorn/templates/daemonset-sa.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager name: longhorn-backend namespace: longhorn-system spec: type: ClusterIP selector: app: longhorn-manager ports: - name: manager port: 9500 targetPort: manager --- # Source: longhorn/templates/deployment-ui.yaml kind: Service apiVersion: v1 metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-frontend namespace: longhorn-system spec: type: ClusterIP selector: app: longhorn-ui ports: - name: http port: 80 targetPort: http nodePort: null --- # Source: longhorn/templates/services.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-admission-webhook name: longhorn-admission-webhook namespace: longhorn-system spec: type: ClusterIP selector: longhorn.io/admission-webhook: longhorn-admission-webhook ports: - name: admission-webhook port: 9502 targetPort: admission-wh --- # Source: longhorn/templates/services.yaml apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-recovery-backend name: longhorn-recovery-backend namespace: longhorn-system spec: type: ClusterIP selector: longhorn.io/recovery-backend: longhorn-recovery-backend ports: - name: recovery-backend port: 9503 targetPort: recov-backend --- # Source: longhorn/templates/daemonset-sa.yaml apiVersion: apps/v1 kind: DaemonSet metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager name: longhorn-manager namespace: longhorn-system spec: selector: matchLabels: app: longhorn-manager template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-manager spec: containers: - name: longhorn-manager image: docker.io/longhornio/longhorn-manager:master-head imagePullPolicy: IfNotPresent securityContext: privileged: true command: - longhorn-manager - -d - daemon - --engine-image - "docker.io/longhornio/longhorn-engine:master-head" - --instance-manager-image - "docker.io/longhornio/longhorn-instance-manager:master-head" - --share-manager-image - "docker.io/longhornio/longhorn-share-manager:master-head" - --backing-image-manager-image - "docker.io/longhornio/backing-image-manager:master-head" - --support-bundle-manager-image - "docker.io/longhornio/support-bundle-kit:v0.0.79" - --manager-image - "docker.io/longhornio/longhorn-manager:master-head" - --service-account - longhorn-service-account - --upgrade-version-check ports: - containerPort: 9500 name: manager - containerPort: 9502 name: admission-wh - containerPort: 9503 name: recov-backend readinessProbe: httpGet: path: /v1/healthz port: 9502 scheme: HTTPS volumeMounts: - name: boot mountPath: /host/boot/ readOnly: true - name: dev mountPath: /host/dev/ - name: proc mountPath: /host/proc/ readOnly: true - name: etc mountPath: /host/etc/ readOnly: true - name: longhorn mountPath: /var/lib/longhorn/ mountPropagation: Bidirectional - name: longhorn-grpc-tls mountPath: /tls-files/ env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: pre-pull-share-manager-image imagePullPolicy: IfNotPresent image: docker.io/longhornio/longhorn-share-manager:master-head command: ["sh", "-c", "echo share-manager image pulled && sleep infinity"] volumes: - name: boot hostPath: path: /boot/ - name: dev hostPath: path: /dev/ - name: proc hostPath: path: /proc/ - name: etc hostPath: path: /etc/ - name: longhorn hostPath: path: /var/lib/longhorn/ - name: longhorn-grpc-tls secret: secretName: longhorn-grpc-tls optional: true priorityClassName: "longhorn-critical" serviceAccountName: longhorn-service-account updateStrategy: rollingUpdate: maxUnavailable: 100% --- # Source: longhorn/templates/deployment-driver.yaml apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-driver-deployer namespace: longhorn-system labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev spec: replicas: 1 selector: matchLabels: app: longhorn-driver-deployer template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-driver-deployer spec: initContainers: - name: wait-longhorn-manager image: docker.io/longhornio/longhorn-manager:master-head command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer image: docker.io/longhornio/longhorn-manager:master-head imagePullPolicy: IfNotPresent command: - longhorn-manager - -d - deploy-driver - --manager-image - "docker.io/longhornio/longhorn-manager:master-head" - --manager-url - http://longhorn-backend:9500/v1 env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: SERVICE_ACCOUNT valueFrom: fieldRef: fieldPath: spec.serviceAccountName - name: CSI_ATTACHER_IMAGE value: "docker.io/longhornio/csi-attacher:v4.10.0-20251226" - name: CSI_PROVISIONER_IMAGE value: "docker.io/longhornio/csi-provisioner:v5.3.0-20251226" - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE value: "docker.io/longhornio/csi-node-driver-registrar:v2.15.0-20251226" - name: CSI_RESIZER_IMAGE value: "docker.io/longhornio/csi-resizer:v2.0.0-20251226" - name: CSI_SNAPSHOTTER_IMAGE value: "docker.io/longhornio/csi-snapshotter:v8.4.0-20251226" - name: CSI_LIVENESS_PROBE_IMAGE value: "docker.io/longhornio/livenessprobe:v2.17.0-20251226" priorityClassName: "longhorn-critical" serviceAccountName: longhorn-service-account securityContext: runAsUser: 0 --- # Source: longhorn/templates/deployment-ui.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui name: longhorn-ui namespace: longhorn-system spec: replicas: 2 selector: matchLabels: app: longhorn-ui template: metadata: labels: app.kubernetes.io/name: longhorn app.kubernetes.io/instance: longhorn app.kubernetes.io/version: v1.9.0-dev app: longhorn-ui spec: serviceAccountName: longhorn-ui-service-account affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - longhorn-ui topologyKey: kubernetes.io/hostname weight: 1 containers: - name: longhorn-ui image: docker.io/longhornio/longhorn-ui:master-head imagePullPolicy: IfNotPresent volumeMounts: - name: nginx-cache mountPath: /var/cache/nginx/ - name: nginx-config mountPath: /var/config/nginx/ - name: var-run mountPath: /var/run/ ports: - containerPort: 8000 name: http env: - name: LONGHORN_MANAGER_IP value: "http://longhorn-backend:9500" - name: LONGHORN_UI_PORT value: "8000" volumes: - emptyDir: {} name: nginx-cache - emptyDir: {} name: nginx-config - emptyDir: {} name: var-run priorityClassName: "longhorn-critical" --- # Source: longhorn/templates/validate-psp-install.yaml # ================================================ FILE: deploy/podsecuritypolicy.yaml ================================================ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: longhorn-psp spec: privileged: true allowPrivilegeEscalation: true requiredDropCapabilities: - NET_RAW allowedCapabilities: - SYS_ADMIN hostNetwork: false hostIPC: false hostPID: true runAsUser: rule: RunAsAny seLinux: rule: RunAsAny fsGroup: rule: RunAsAny supplementalGroups: rule: RunAsAny volumes: - configMap - downwardAPI - emptyDir - secret - projected - hostPath --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: longhorn-psp-role namespace: longhorn-system rules: - apiGroups: - policy resources: - podsecuritypolicies verbs: - use resourceNames: - longhorn-psp --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: longhorn-psp-binding namespace: longhorn-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: longhorn-psp-role subjects: - kind: ServiceAccount name: longhorn-service-account namespace: longhorn-system - kind: ServiceAccount name: default namespace: longhorn-system ================================================ FILE: deploy/prerequisite/longhorn-cifs-installation.yaml ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: longhorn-cifs-installation namespace: longhorn-system labels: app: longhorn-cifs-installation annotations: command: &cmd OS=$(grep -E "^ID_LIKE=" /etc/os-release | cut -d '=' -f 2); if [[ -z "${OS}" ]]; then OS=$(grep -E "^ID=" /etc/os-release | cut -d '=' -f 2); fi; if [[ "${OS}" == *"debian"* ]]; then sudo apt-get update -q -y && sudo apt-get install -q -y cifs-utils; elif [[ "${OS}" == *"suse"* ]]; then sudo zypper --gpg-auto-import-keys -q refresh && sudo zypper --gpg-auto-import-keys -q install -y cifs-utils; else sudo yum makecache -q -y && sudo yum --setopt=tsflags=noscripts install -q -y cifs-utils; fi && if [ $? -eq 0 ]; then echo "cifs install successfully"; else echo "cifs utilities install failed error code $?"; fi spec: selector: matchLabels: app: longhorn-cifs-installation template: metadata: labels: app: longhorn-cifs-installation spec: hostNetwork: true hostPID: true initContainers: - name: cifs-installation command: - nsenter - --mount=/proc/1/ns/mnt - -- - bash - -c - *cmd image: alpine:3.12 securityContext: privileged: true containers: - name: sleep image: registry.k8s.io/pause:3.1 updateStrategy: type: RollingUpdate ================================================ FILE: deploy/prerequisite/longhorn-gke-cos-node-agent.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: name: longhorn-gke-cos-node-agent-entrypoint namespace: longhorn-system labels: app: longhorn-gke-cos-node data: entrypoint.sh: | #!/bin/bash set -euo pipefail # Define default directories HOST_MOUNT_DIR="${HOST_MOUNT_DIR:-/host}" KUBERNETES_ROOTFS="${KUBERNETES_ROOTFS:-/home/kubernetes/containerized_mounter/rootfs}" KUBERNETES_MOUNT_DIR="${HOST_MOUNT_DIR}${KUBERNETES_ROOTFS}" LONGHORN_DATA_PATHS="${LONGHORN_DATA_PATHS:-/var/lib/longhorn}" IFS=',' read -ra LONGHORN_DATA_DIRS <<< "$LONGHORN_DATA_PATHS" # Split comma-separated dirs # Function to mount the Longhorn data directory on the host mount_longhorn_data_dir_on_host() { local _longhorn_data_dir="$1" if is_mounted_on_host "${_longhorn_data_dir}"; then echo "Longhorn data directory ${_longhorn_data_dir} is already mounted" else echo "Mounting Longhorn data directory ${_longhorn_data_dir} on the host" chroot "${HOST_MOUNT_DIR}" mkdir -p "${_longhorn_data_dir}" chroot "${KUBERNETES_MOUNT_DIR}" mkdir -p "${_longhorn_data_dir}" nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" mount --rbind "${_longhorn_data_dir}" "${KUBERNETES_ROOTFS}${_longhorn_data_dir}" nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" mount --make-shared "${KUBERNETES_ROOTFS}${_longhorn_data_dir}" nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" mount -o remount,exec "${_longhorn_data_dir}" fi } # Function to check if a directory is already mounted is_mounted_on_host() { nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" findmnt --noheadings --output TARGET "$1" } # Function to check if a kernel module is loaded is_module_loaded_on_host() { local _module="$1" nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" lsmod | grep -q "${_module}" } # Function to load the iscsi_tcp kernel module on the host load_iscsi_tcp_module_on_host() { if is_module_loaded_on_host "iscsi_tcp"; then echo "iscsi_tcp kernel module is already loaded" else echo "Loading iscsi_tcp kernel module" nsenter --mount="${HOST_MOUNT_DIR}/proc/1/ns/mnt" modprobe iscsi_tcp fi } # Function to install and start open-iscsi install_and_start_iscsid() { echo "Installing and starting open-iscsi" zypper install -y open-iscsi /sbin/iscsid } # Mount the Longhorn data directories for LONGHORN_DATA_DIR in "${LONGHORN_DATA_DIRS[@]}"; do mount_longhorn_data_dir_on_host "${LONGHORN_DATA_DIR}" done install_and_start_iscsid load_iscsi_tcp_module_on_host echo "Complete!" echo "Keep the container running for iSCSI daemon" sleep infinity --- apiVersion: apps/v1 kind: DaemonSet metadata: name: longhorn-gke-cos-node-agent namespace: longhorn-system labels: app: longhorn-gke-cos-node spec: selector: matchLabels: app: longhorn-gke-cos-node updateStrategy: type: RollingUpdate template: metadata: labels: name: longhorn-gke-cos-node-agent app: longhorn-gke-cos-node spec: volumes: - name: host-mount hostPath: path: / - name: entrypoint configMap: name: longhorn-gke-cos-node-agent-entrypoint defaultMode: 0744 hostNetwork: true containers: - image: registry.suse.com/bci/bci-base:15.5 name: node-agent command: ["/scripts/entrypoint.sh"] env: - name: HOST_MOUNT_DIR value: /host - name: KUBERNETES_ROOTFS value: /home/kubernetes/containerized_mounter/rootfs - name: LONGHORN_DATA_PATHS # Comma-separated list of Longhorn data paths # For example: /var/lib/longhorn1,/var/lib/longhorn2 value: /var/lib/longhorn securityContext: capabilities: add: - SYS_MODULE privileged: true volumeMounts: - name: host-mount mountPath: /host - name: entrypoint mountPath: /scripts readinessProbe: exec: command: - "/bin/bash" - "-c" - | nsenter --mount=${HOST_MOUNT_DIR}/proc/1/ns/mnt pgrep -x iscsid && \ nsenter --mount=${HOST_MOUNT_DIR}/proc/1/ns/mnt lsmod | grep -q iscsi_tcp initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 livenessProbe: exec: command: - "/bin/bash" - "-c" - | nsenter --mount=${HOST_MOUNT_DIR}/proc/1/ns/mnt pgrep -x iscsid && \ nsenter --mount=${HOST_MOUNT_DIR}/proc/1/ns/mnt lsmod | grep -q iscsi_tcp initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ================================================ FILE: deploy/prerequisite/longhorn-iscsi-selinux-workaround.yaml ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: longhorn-iscsi-selinux-workaround namespace: longhorn-system labels: app: longhorn-iscsi-selinux-workaround annotations: command: &cmd if ! rpm -q policycoreutils > /dev/null 2>&1; then echo "failed to apply workaround; only applicable in Fedora based distros with SELinux enabled"; exit; elif cd /tmp && echo '(allow iscsid_t self (capability (dac_override)))' > local_longhorn.cil && semodule -vi local_longhorn.cil && rm -f local_longhorn.cil; then echo "applied workaround successfully"; else echo "failed to apply workaround; error code $?"; fi spec: selector: matchLabels: app: longhorn-iscsi-selinux-workaround template: metadata: labels: app: longhorn-iscsi-selinux-workaround spec: hostPID: true initContainers: - name: iscsi-selinux-workaround command: - nsenter - --mount=/proc/1/ns/mnt - -- - bash - -c - *cmd image: alpine:3.17 securityContext: privileged: true containers: - name: sleep image: registry.k8s.io/pause:3.1 updateStrategy: type: RollingUpdate ================================================ FILE: deploy/upgrade_responder_server/README.md ================================================ # Upgrade Responder Helm Chart This directory contains the helm values for the Longhorn upgrade responder server. The values are in the file `./chart-values.yaml`. When you update the content of `./chart-values.yaml`, automation pipeline will update the Longhorn upgrade responder. Information about the source chart is in `chart.yaml`. See [dev/upgrade-responder](../../dev/upgrade-responder/README.md) for manual deployment steps. ================================================ FILE: deploy/upgrade_responder_server/chart-values.yaml ================================================ # Specify the name of the application that is using this Upgrade Responder server # This will be used to create a database named _upgrade_responder # in the InfluxDB to store all data for this Upgrade Responder # The name must be in snake case format applicationName: longhorn image: repository: longhornio/upgrade-responder tag: longhorn-head pullPolicy: Always secret: name: upgrade-responder-secret # Set this to false if you don't want to manage these secrets with helm managed: false resources: limits: cpu: 400m memory: 512Mi requests: cpu: 200m memory: 256Mi flags: scarfEndpoint: "https://longhorn.gateway.scarf.sh/{version}" # This configmap contains information about the latest release # of the application that is using this Upgrade Responder configMap: responseConfig: |- { "versions": [ { "name": "v1.3.3", "releaseDate": "2023-04-19T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.4.4", "releaseDate": "2023-10-26T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.5.5", "releaseDate": "2024-04-19T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.6.4", "releaseDate": "2025-01-02T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.7.3", "releaseDate": "2025-02-19T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.8.2", "releaseDate": "2025-06-08T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.9.2", "releaseDate": "2025-09-24T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.10.2", "releaseDate": "2026-01-28T00:00:00Z", "tags": [ "stable" ] }, { "name": "v1.11.0", "releaseDate": "2026-01-29T00:00:00Z", "tags": [ "latest" ] } ] } requestSchema: |- { "appVersionSchema": { "dataType": "string", "maxLen": 200 }, "extraTagInfoSchema": { "hostArch": { "dataType": "string", "maxLen": 200 }, "hostKernelRelease": { "dataType": "string", "maxLen": 200 }, "hostOsDistro": { "dataType": "string", "maxLen": 200 }, "kubernetesNodeProvider": { "dataType": "string", "maxLen": 200 }, "kubernetesVersion": { "dataType": "string", "maxLen": 200 }, "longhornImageRegistry": { "dataType": "string", "maxLen": 200 }, "longhornSettingAllowRecurringJobWhileVolumeDetached": { "dataType": "string", "maxLen": 200 }, "longhornSettingAllowVolumeCreationWithDegradedAvailability": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoCleanupSystemGeneratedSnapshot": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoDeletePodWhenVolumeDetachedUnexpectedly": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoSalvage": { "dataType": "string", "maxLen": 200 }, "longhornSettingBackupCompressionMethod": { "dataType": "string", "maxLen": 200 }, "longhornSettingBackupTarget": { "dataType": "string", "maxLen": 200 }, "longhornSettingCrdApiVersion": { "dataType": "string", "maxLen": 200 }, "longhornSettingCreateDefaultDiskLabeledNodes": { "dataType": "string", "maxLen": 200 }, "longhornSettingDefaultDataLocality": { "dataType": "string", "maxLen": 200 }, "longhornSettingDisableRevisionCounter": { "dataType": "string", "maxLen": 200 }, "longhornSettingDisableSchedulingOnCordonedNode": { "dataType": "string", "maxLen": 200 }, "longhornSettingFastReplicaRebuildEnabled": { "dataType": "string", "maxLen": 200 }, "longhornSettingFreezeFilesystemForSnapshot": { "dataType": "string", "maxLen": 200 }, "longhornSettingKubernetesClusterAutoscalerEnabled": { "dataType": "string", "maxLen": 200 }, "longhornSettingNodeDownPodDeletionPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingNodeDrainPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingOrphanAutoDeletion": { "dataType": "string", "maxLen": 200 }, "longhornSettingOrphanResourceAutoDeletion": { "dataType": "string", "maxLen": 200 }, "longhornSettingPriorityClass": { "dataType": "string", "maxLen": 200 }, "longhornSettingRegistrySecret": { "dataType": "string", "maxLen": 200 }, "longhornSettingRemoveSnapshotsDuringFilesystemTrim": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaAutoBalance": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaZoneSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaDiskSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingRestoreVolumeRecurringJobs": { "dataType": "string", "maxLen": 200 }, "longhornSettingRwxVolumeFastFailover": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrity": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrityCronjob": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrityImmediateCheckAfterSnapshotCreation": { "dataType": "string", "maxLen": 200 }, "longhornSettingStorageNetwork": { "dataType": "string", "maxLen": 200 }, "longhornSettingEndpointNetworkForRWXVolume": { "dataType": "string", "maxLen": 200 }, "longhornSettingSystemManagedComponentsNodeSelector": { "dataType": "string", "maxLen": 200 }, "longhornSettingSystemManagedPodsImagePullPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingTaintToleration": { "dataType": "string", "maxLen": 200 }, "longhornSettingV1DataEngine": { "dataType": "string", "maxLen": 200 }, "longhornSettingV2DataEngine": { "dataType": "string", "maxLen": 200 } }, "extraFieldInfoSchema": { "longhornBackingImageCount": { "dataType": "float" }, "longhornBackupTargetAzblobCount": { "dataType": "float" }, "longhornBackupTargetCifsCount": { "dataType": "float" }, "longhornBackupTargetNfsCount": { "dataType": "float" }, "longhornBackupTargetS3Count": { "dataType": "float" }, "longhornBackupTargetUnknownCount": { "dataType": "float" }, "longhornDiskBlockCount": { "dataType": "float" }, "longhornDiskFilesystemCount": { "dataType": "float" }, "longhornInstanceManagerAverageCpuUsageMilliCores": { "dataType": "float" }, "longhornInstanceManagerAverageMemoryUsageBytes": { "dataType": "float" }, "longhornManagerAverageCpuUsageMilliCores": { "dataType": "float" }, "longhornManagerAverageMemoryUsageBytes": { "dataType": "float" }, "longhornNamespaceUid": { "dataType": "string", "maxLen": 200 }, "longhornNodeCount": { "dataType": "float" }, "longhornNodeDiskHDDCount": { "dataType": "float" }, "longhornNodeDiskNVMeCount": { "dataType": "float" }, "longhornNodeDiskSSDCount": { "dataType": "float" }, "longhornOrphanCount": { "dataType": "float" }, "longhornSettingBackingImageCleanupWaitInterval": { "dataType": "float" }, "longhornSettingBackingImageRecoveryWaitInterval": { "dataType": "float" }, "longhornSettingBackupConcurrentLimit": { "dataType": "float" }, "longhornSettingBackupstorePollInterval": { "dataType": "float" }, "longhornSettingBackupBlockSize": { "dataType": "float" }, "longhornSettingConcurrentAutomaticEngineUpgradePerNodeLimit": { "dataType": "float" }, "longhornSettingConcurrentReplicaRebuildPerNodeLimit": { "dataType": "float" }, "longhornSettingConcurrentVolumeBackupRestorePerNodeLimit": { "dataType": "float" }, "longhornSettingDefaultReplicaCount": { "dataType": "float" }, "longhornSettingEngineReplicaTimeout": { "dataType": "float" }, "longhornSettingFailedBackupTtl": { "dataType": "float" }, "longhornSettingReplicaRebuildingBandwidthLimit": { "dataType": "float" }, "longhornSettingGuaranteedInstanceManagerCpu": { "dataType": "float" }, "longhornSettingRecurringFailedJobsHistoryLimit": { "dataType": "float" }, "longhornSettingRecurringSuccessfulJobsHistoryLimit": { "dataType": "float" }, "longhornSettingReplicaFileSyncHttpClientTimeout": { "dataType": "float" }, "longhornSettingReplicaReplenishmentWaitInterval": { "dataType": "float" }, "longhornSettingRestoreConcurrentLimit": { "dataType": "float" }, "longhornSettingStorageMinimalAvailablePercentage": { "dataType": "float" }, "longhornSettingStorageOverProvisioningPercentage": { "dataType": "float" }, "longhornSettingStorageReservedPercentageForDefaultDisk": { "dataType": "float" }, "longhornSettingSupportBundleFailedHistoryLimit": { "dataType": "float" }, "longhornVolumeAccessModeRwoCount": { "dataType": "float" }, "longhornVolumeAccessModeRwxCount": { "dataType": "float" }, "longhornVolumeAccessModeUnknownCount": { "dataType": "float" }, "longhornVolumeAverageActualSizeBytes": { "dataType": "float" }, "longhornVolumeAverageNumberOfReplicas": { "dataType": "float" }, "longhornVolumeAverageSizeBytes": { "dataType": "float" }, "longhornVolumeAverageSnapshotCount": { "dataType": "float" }, "longhornVolumeBackendStoreDriverV1Count": { "dataType": "float" }, "longhornVolumeBackendStoreDriverV2Count": { "dataType": "float" }, "longhornVolumeDataLocalityBestEffortCount": { "dataType": "float" }, "longhornVolumeDataLocalityDisabledCount": { "dataType": "float" }, "longhornVolumeDataLocalityStrictLocalCount": { "dataType": "float" }, "longhornVolumeEncryptedTrueCount": { "dataType": "float" }, "longhornVolumeEncryptedFalseCount": { "dataType": "float" }, "longhornVolumeFreezeFilesystemForSnapshotTrueCount": { "dataType": "float" }, "longhornVolumeFrontendBlockdevCount": { "dataType": "float" }, "longhornVolumeFrontendIscsiCount": { "dataType": "float" }, "longhornVolumeNumberOfReplicas": { "dataType": "float" }, "longhornVolumeNumberOfSnapshots": { "dataType": "float" }, "longhornVolumeReplicaAutoBalanceDisabledCount": { "dataType": "float" }, "longhornVolumeReplicaSoftAntiAffinityFalseCount": { "dataType": "float" }, "longhornVolumeReplicaZoneSoftAntiAffinityTrueCount": { "dataType": "float" }, "longhornVolumeReplicaDiskSoftAntiAffinityTrueCount": { "dataType": "float" }, "longhornVolumeRestoreVolumeRecurringJobFalseCount": { "dataType": "float" }, "longhornVolumeSnapshotDataIntegrityDisabledCount": { "dataType": "float" }, "longhornVolumeSnapshotDataIntegrityFastCheckCount": { "dataType": "float" }, "longhornVolumeUnmapMarkSnapChainRemovedFalseCount": { "dataType": "float" }, "longhornSettingOrphanResourceAutoDeletionGracePeriod": { "dataType": "float" }, "longhornSettingSnapshotHeavyTaskConcurrentLimit": { "dataType": "float" } } } ================================================ FILE: deploy/upgrade_responder_server/chart.yaml ================================================ url: https://github.com/longhorn/upgrade-responder.git commit: e16207218faf9203c2bf1270896128781c605599 releaseName: longhorn-upgrade-responder namespace: longhorn-upgrade-responder ================================================ FILE: dev/scale-test/.gitignore ================================================ # ignores all goland project folders and files .idea *.iml *.ipr # ignore output folder out tmp results # ignore kubeconfig kubeconfig ================================================ FILE: dev/scale-test/README.md ================================================ ## Overview scale-test is a collection of developer scripts that are used for scaling a cluster to a certain amount of volumes while monitoring the time required to complete these actions. `sample.sh` can be used to quickly see how long it takes for the requested amount of volumes to be up and usable. `scale-test.py` can be used to create the amount of requested statefulsets based on the `statefulset.yaml` template, as well as retrieve detailed timing information per volume. ### scale-test.py scale-test.py watches `pod`, `pvc`, `va` events (ADDED, MODIFIED, DELETED). Based on that information we can calculate the time of actions for each individual pod. In additional scale-test.py can also be used to create a set of statefulset deployment files. based on the `statefulset.yaml` with the following VARIABLES substituted based on the current sts index. `@NODE_NAME@` - schedule each sts on a dedicated node `@STS_NAME@` - also used for the volume-name make sure to set the correct CONSTANT values in scale-test.py before running. ### sample.sh sample.sh can be used to scale to a requested amount of volumes based on the existing statefulsets and node count for the current cluster. One can pass the requested amount of volumes as well as the node count of the current cluster. example for 1000 volumes and 100 nodes: `./sample.sh 1000 100` this expects there to be a statefulset deployment for each node. ================================================ FILE: dev/scale-test/sample.sh ================================================ #!/usr/bin/env bash requested=${1:-0} node_count=${2:-1} required_scale=$((requested / node_count)) now=$(date) ready=$(kubectl get pods -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY:status.containerStatuses[*].ready | grep -c true) echo "$ready -- $now - start state" cmd=$(kubectl scale --replicas="$required_scale" statefulset --all) echo "$cmd" while [ "$ready" -ne "$requested" ]; do sleep 60 now=$(date) ready=$(kubectl get pods -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY:status.containerStatuses[*].ready | grep -c true) echo "$ready -- $now - delta:" done echo "$requested -- $now - done state" ================================================ FILE: dev/scale-test/scale-test.py ================================================ import sys import asyncio import logging from pathlib import Path from kubernetes import client, config, watch NAMESPACE = "default" NODE_PREFIX = "jmoody-work" NODE_COUNT = 100 TEMPLATE_FILE = "statefulset.yaml" KUBE_CONFIG = None KUBE_CONTEXT = None # KUBE_CONFIG = "kubeconfig" # KUBE_CONTEXT = "jmoody-test-jmoody-control2" def create_sts_deployment(count): # @NODE_NAME@ - schedule each sts on a dedicated node # @STS_NAME@ - also used for the volume-name # create 100 stateful-sets for i in range(count): create_sts_yaml(i + 1) def create_sts_yaml(index): content = Path(TEMPLATE_FILE).read_text() content = content.replace("@NODE_NAME@", NODE_PREFIX + str(index)) content = content.replace("@STS_NAME@", "sts" + str(index)) file = Path("out/sts" + str(index) + ".yaml") file.parent.mkdir(parents=True, exist_ok=True) file.write_text(content) async def watch_pods_async(): log = logging.getLogger('pod_events') log.setLevel(logging.INFO) v1 = client.CoreV1Api() w = watch.Watch() for event in w.stream(v1.list_namespaced_pod, namespace=NAMESPACE): process_pod_event(log, event) await asyncio.sleep(0) def process_pod_event(log, event): log.info("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) if 'ADDED' in event['type']: pass elif 'DELETED' in event['type']: pass else: pass async def watch_pvc_async(): log = logging.getLogger('pvc_events') log.setLevel(logging.INFO) v1 = client.CoreV1Api() w = watch.Watch() for event in w.stream(v1.list_namespaced_persistent_volume_claim, namespace=NAMESPACE): process_pvc_event(log, event) await asyncio.sleep(0) def process_pvc_event(log, event): log.info("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) if 'ADDED' in event['type']: pass elif 'DELETED' in event['type']: pass else: pass async def watch_va_async(): log = logging.getLogger('va_events') log.setLevel(logging.INFO) storage = client.StorageV1Api() w = watch.Watch() for event in w.stream(storage.list_volume_attachment): process_va_event(log, event) await asyncio.sleep(0) def process_va_event(log, event): log.info("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) if 'ADDED' in event['type']: pass elif 'DELETED' in event['type']: pass else: pass if __name__ == '__main__': # create the sts deployment files create_sts_deployment(NODE_COUNT) # setup the monitor log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format) config.load_kube_config(config_file=KUBE_CONFIG, context=KUBE_CONTEXT) logging.info("scale-test started") # datastructures to keep track of the timings # TODO: process events and keep track of the results # results should be per pod/volume # information to keep track: pod index per sts # volume-creation time per pod # volume-attach time per pod # volume-detach time per pod pvc_to_va_map = dict() pvc_to_pod_map = dict() results = dict() # start async event_loop event_loop = asyncio.get_event_loop() event_loop.create_task(watch_pods_async()) event_loop.create_task(watch_pvc_async()) event_loop.create_task(watch_va_async()) event_loop.run_forever() logging.info("scale-test-finished") ================================================ FILE: dev/scale-test/statefulset.yaml ================================================ apiVersion: apps/v1 kind: StatefulSet metadata: name: @STS_NAME@ spec: replicas: 0 serviceName: @STS_NAME@ selector: matchLabels: app: @STS_NAME@ template: metadata: labels: app: @STS_NAME@ spec: nodeName: @NODE_NAME@ restartPolicy: Always terminationGracePeriodSeconds: 10 containers: - name: '@STS_NAME@' image: 'busybox:latest' command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 5 ; done"] livenessProbe: exec: command: - ls - /mnt/@STS_NAME@ initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: @STS_NAME@ mountPath: /mnt/@STS_NAME@ volumeClaimTemplates: - metadata: name: @STS_NAME@ spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "longhorn" resources: requests: storage: 1Gi ================================================ FILE: dev/scripts/lm-update.sh ================================================ #!/usr/bin/env bash #set -x set -e username=$1 if [ "$username" == "" ] then echo DockerHub username is required exit 1 fi update=$2 project="longhorn-manager" base="${GOPATH}/src/github.com/longhorn/longhorn-manager" yaml=${base}"/deploy/install/02-components/01-manager.yaml" driver_yaml=${base}"/deploy/install/02-components/04-driver.yaml" latest=`cat ${base}/bin/latest_image` private=`sed "s/longhornio/${username}/g" ${base}/bin/latest_image` echo Latest image ${latest} echo Latest private image ${private} docker tag ${latest} ${private} docker push ${private} escaped_private=${private//\//\\\/} sed -i "s/image\:\ .*\/${project}:.*/image\:\ ${escaped_private}/g" $yaml sed -i "s/-\ .*\/${project}:.*/-\ ${escaped_private}/g" $yaml sed -i "s/imagePullPolicy\:\ .*/imagePullPolicy\:\ Always/g" $yaml sed -i "s/image\:\ .*\/${project}:.*/image\:\ ${escaped_private}/g" $driver_yaml sed -i "s/-\ .*\/${project}:.*/-\ ${escaped_private}/g" $driver_yaml sed -i "s/imagePullPolicy\:\ .*/imagePullPolicy\:\ Always/g" $driver_yaml set +e if [ "$update" == "" ] then kubectl delete -f $yaml kubectl create -f $yaml kubectl delete -f $driver_yaml kubectl create -f $driver_yaml fi ================================================ FILE: dev/scripts/update-image-pull-policy.sh ================================================ #!/usr/bin/env bash NS=longhorn-system KINDS="daemonset deployments" function patch_kind { kind=$1 list=$(kubectl -n $NS get $kind -o name) for obj in $list do echo Updating $obj to imagePullPolicy: Always name=${obj##*/} kubectl -n $NS patch $obj -p '{"spec": {"template": {"spec":{"containers":[{"name":"'$name'","imagePullPolicy":"Always"}]}}}}' done } for kind in $KINDS do patch_kind $kind done echo "Warning: Make sure check and wait for all pods running again!" echo "Current status: (CTRL-C to exit)" kubectl get pods -w -n longhorn-system ================================================ FILE: dev/upgrade-responder/README.md ================================================ ## Overview ### Install 1. Install Longhorn. 1. Install Longhorn [upgrade-responder](https://github.com/longhorn/upgrade-responder) stack. ```bash ./install.sh ``` Sample output: ```shell secret/influxdb-creds created persistentvolumeclaim/influxdb created deployment.apps/influxdb created service/influxdb created Deployment influxdb is running. Cloning into 'upgrade-responder'... remote: Enumerating objects: 1077, done. remote: Counting objects: 100% (1076/1076), done. remote: Compressing objects: 100% (454/454), done. remote: Total 1077 (delta 573), reused 1049 (delta 565), pack-reused 1 Receiving objects: 100% (1077/1077), 55.01 MiB | 18.10 MiB/s, done. Resolving deltas: 100% (573/573), done. Release "longhorn-upgrade-responder" does not exist. Installing it now. NAME: longhorn-upgrade-responder LAST DEPLOYED: Thu May 11 00:42:44 2023 NAMESPACE: default STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: 1. Get the Upgrade Responder server URL by running these commands: export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=upgrade-responder,app.kubernetes.io/instance=longhorn-upgrade-responder" -o jsonpath="{.items[0].metadata.name}") kubectl port-forward $POD_NAME 8080:8314 --namespace default echo "Upgrade Responder server URL is http://127.0.0.1:8080" Deployment longhorn-upgrade-responder is running. persistentvolumeclaim/grafana-pvc created deployment.apps/grafana created service/grafana created Deployment grafana is running. [Upgrade Checker] URL : http://longhorn-upgrade-responder.default.svc.cluster.local:8314/v1/checkupgrade [InfluxDB] URL : http://influxdb.default.svc.cluster.local:8086 Database : longhorn_upgrade_responder Username : root Password : root [Grafana] Dashboard : http://1.2.3.4:30864 Username : admin Password : admin ``` ================================================ FILE: dev/upgrade-responder/install.sh ================================================ #!/usr/bin/env bash UPGRADE_RESPONDER_REPO="https://github.com/longhorn/upgrade-responder.git" UPGRADE_RESPONDER_REPO_BRANCH="master" UPGRADE_RESPONDER_VALUE_YAML="upgrade-responder-value.yaml" UPGRADE_RESPONDER_IMAGE_REPO="longhornio/upgrade-responder" UPGRADE_RESPONDER_IMAGE_TAG="longhorn-head" INFLUXDB_URL="http://influxdb.default.svc.cluster.local:8086" APP_NAME="longhorn" DEPLOYMENT_TIMEOUT_SEC=300 DEPLOYMENT_WAIT_INTERVAL_SEC=5 temp_dir=$(mktemp -d) trap 'rm -rf "${temp_dir}"' EXIT # -f because packed Git files (.pack, .idx) are write protected. cp -a ./* ${temp_dir} cd ${temp_dir} wait_for_deployment() { local deployment_name="$1" local start_time=$(date +%s) while true; do status=$(kubectl rollout status deployment/${deployment_name}) if [[ ${status} == *"successfully rolled out"* ]]; then echo "Deployment ${deployment_name} is running." break fi elapsed_secs=$(($(date +%s) - ${start_time})) if [[ ${elapsed_secs} -ge ${timeout_secs} ]]; then echo "Timed out waiting for deployment ${deployment_name} to be running." exit 1 fi echo "Deployment ${deployment_name} is not running yet. Waiting..." sleep ${DEPLOYMENT_WAIT_INTERVAL_SEC} done } install_influxdb() { kubectl apply -f ./manifests/influxdb.yaml wait_for_deployment "influxdb" } install_grafana() { kubectl apply -f ./manifests/grafana.yaml wait_for_deployment "grafana" } install_upgrade_responder() { cat << EOF > ${UPGRADE_RESPONDER_VALUE_YAML} applicationName: ${APP_NAME} secret: name: upgrade-responder-secrets managed: true influxDBUrl: "${INFLUXDB_URL}" influxDBUser: "root" influxDBPassword: "root" flags: scarfEndpoint: "https://longhorn.gateway.scarf.sh/{version}" configMap: responseConfig: |- { "versions": [{ "name": "v1.0.0", "releaseDate": "2020-05-18T12:30:00Z", "tags": ["latest"] }] } requestSchema: |- { "appVersionSchema": { "dataType": "string", "maxLen": 200 }, "extraTagInfoSchema": { "hostArch": { "dataType": "string", "maxLen": 200 }, "hostKernelRelease": { "dataType": "string", "maxLen": 200 }, "hostOsDistro": { "dataType": "string", "maxLen": 200 }, "kubernetesNodeProvider": { "dataType": "string", "maxLen": 200 }, "kubernetesVersion": { "dataType": "string", "maxLen": 200 }, "longhornImageRegistry": { "dataType": "string", "maxLen": 200 }, "longhornSettingAllowRecurringJobWhileVolumeDetached": { "dataType": "string", "maxLen": 200 }, "longhornSettingAllowVolumeCreationWithDegradedAvailability": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoCleanupSystemGeneratedSnapshot": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoDeletePodWhenVolumeDetachedUnexpectedly": { "dataType": "string", "maxLen": 200 }, "longhornSettingAutoSalvage": { "dataType": "string", "maxLen": 200 }, "longhornSettingBackupCompressionMethod": { "dataType": "string", "maxLen": 200 }, "longhornSettingCrdApiVersion": { "dataType": "string", "maxLen": 200 }, "longhornSettingCreateDefaultDiskLabeledNodes": { "dataType": "string", "maxLen": 200 }, "longhornSettingDefaultDataLocality": { "dataType": "string", "maxLen": 200 }, "longhornSettingDisableRevisionCounter": { "dataType": "string", "maxLen": 200 }, "longhornSettingDisableSchedulingOnCordonedNode": { "dataType": "string", "maxLen": 200 }, "longhornSettingFastReplicaRebuildEnabled": { "dataType": "string", "maxLen": 200 }, "longhornSettingFreezeFilesystemForSnapshot": { "dataType": "string", "maxLen": 200 }, "longhornSettingKubernetesClusterAutoscalerEnabled": { "dataType": "string", "maxLen": 200 }, "longhornSettingNodeDownPodDeletionPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingNodeDrainPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingOrphanResourceAutoDeletion": { "dataType": "string", "maxLen": 200 }, "longhornSettingPriorityClass": { "dataType": "string", "maxLen": 200 }, "longhornSettingRegistrySecret": { "dataType": "string", "maxLen": 200 }, "longhornSettingRemoveSnapshotsDuringFilesystemTrim": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaAutoBalance": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaZoneSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingReplicaDiskSoftAntiAffinity": { "dataType": "string", "maxLen": 200 }, "longhornSettingRestoreVolumeRecurringJobs": { "dataType": "string", "maxLen": 200 }, "longhornSettingRwxVolumeFastFailover": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrity": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrityCronjob": { "dataType": "string", "maxLen": 200 }, "longhornSettingSnapshotDataIntegrityImmediateCheckAfterSnapshotCreation": { "dataType": "string", "maxLen": 200 }, "longhornSettingStorageNetwork": { "dataType": "string", "maxLen": 200 }, "longhornSettingEndpointNetworkForRWXVolume": { "dataType": "string", "maxLen": 200 }, "longhornSettingSystemManagedComponentsNodeSelector": { "dataType": "string", "maxLen": 200 }, "longhornSettingSystemManagedPodsImagePullPolicy": { "dataType": "string", "maxLen": 200 }, "longhornSettingTaintToleration": { "dataType": "string", "maxLen": 200 }, "longhornSettingV1DataEngine": { "dataType": "string", "maxLen": 200 }, "longhornSettingV2DataEngine": { "dataType": "string", "maxLen": 200 } }, "extraFieldInfoSchema": { "longhornBackingImageCount": { "dataType": "float" }, "longhornBackupTargetAzblobCount": { "dataType": "float" }, "longhornBackupTargetCifsCount": { "dataType": "float" }, "longhornBackupTargetNfsCount": { "dataType": "float" }, "longhornBackupTargetS3Count": { "dataType": "float" }, "longhornBackupTargetUnknownCount": { "dataType": "float" }, "longhornDiskBlockCount": { "dataType": "float" }, "longhornDiskFilesystemCount": { "dataType": "float" }, "longhornInstanceManagerAverageCpuUsageMilliCores": { "dataType": "float" }, "longhornInstanceManagerAverageMemoryUsageBytes": { "dataType": "float" }, "longhornManagerAverageCpuUsageMilliCores": { "dataType": "float" }, "longhornManagerAverageMemoryUsageBytes": { "dataType": "float" }, "longhornNamespaceUid": { "dataType": "string", "maxLen": 200 }, "longhornNodeCount": { "dataType": "float" }, "longhornNodeDiskHDDCount": { "dataType": "float" }, "longhornNodeDiskNVMeCount": { "dataType": "float" }, "longhornNodeDiskSSDCount": { "dataType": "float" }, "longhornOrphanCount": { "dataType": "float" }, "longhornSettingBackingImageCleanupWaitInterval": { "dataType": "float" }, "longhornSettingBackingImageRecoveryWaitInterval": { "dataType": "float" }, "longhornSettingBackupConcurrentLimit": { "dataType": "float" }, "longhornSettingBackupstorePollInterval": { "dataType": "float" }, "longhornSettingBackupBlockSize": { "dataType": "float" }, "longhornSettingConcurrentAutomaticEngineUpgradePerNodeLimit": { "dataType": "float" }, "longhornSettingConcurrentReplicaRebuildPerNodeLimit": { "dataType": "float" }, "longhornSettingConcurrentVolumeBackupRestorePerNodeLimit": { "dataType": "float" }, "longhornSettingDefaultReplicaCount": { "dataType": "float" }, "longhornSettingEngineReplicaTimeout": { "dataType": "float" }, "longhornSettingFailedBackupTtl": { "dataType": "float" }, "longhornSettingReplicaRebuildingBandwidthLimit": { "dataType": "float" }, "longhornSettingGuaranteedInstanceManagerCpu": { "dataType": "float" }, "longhornSettingRecurringFailedJobsHistoryLimit": { "dataType": "float" }, "longhornSettingRecurringSuccessfulJobsHistoryLimit": { "dataType": "float" }, "longhornSettingReplicaFileSyncHttpClientTimeout": { "dataType": "float" }, "longhornSettingReplicaReplenishmentWaitInterval": { "dataType": "float" }, "longhornSettingRestoreConcurrentLimit": { "dataType": "float" }, "longhornSettingStorageMinimalAvailablePercentage": { "dataType": "float" }, "longhornSettingStorageOverProvisioningPercentage": { "dataType": "float" }, "longhornSettingStorageReservedPercentageForDefaultDisk": { "dataType": "float" }, "longhornSettingSupportBundleFailedHistoryLimit": { "dataType": "float" }, "longhornVolumeAccessModeRwoCount": { "dataType": "float" }, "longhornVolumeAccessModeRwxCount": { "dataType": "float" }, "longhornVolumeAccessModeUnknownCount": { "dataType": "float" }, "longhornVolumeAverageActualSizeBytes": { "dataType": "float" }, "longhornVolumeAverageNumberOfReplicas": { "dataType": "float" }, "longhornVolumeAverageSizeBytes": { "dataType": "float" }, "longhornVolumeAverageSnapshotCount": { "dataType": "float" }, "longhornVolumeBackendStoreDriverV1Count": { "dataType": "float" }, "longhornVolumeBackendStoreDriverV2Count": { "dataType": "float" }, "longhornVolumeDataLocalityBestEffortCount": { "dataType": "float" }, "longhornVolumeDataLocalityDisabledCount": { "dataType": "float" }, "longhornVolumeDataLocalityStrictLocalCount": { "dataType": "float" }, "longhornVolumeEncryptedTrueCount": { "dataType": "float" }, "longhornVolumeEncryptedFalseCount": { "dataType": "float" }, "longhornFreezeFilesystemForSnapshotTrueCount": { "dataType": "float" }, "longhornVolumeFrontendBlockdevCount": { "dataType": "float" }, "longhornVolumeFrontendIscsiCount": { "dataType": "float" }, "longhornVolumeNumberOfReplicas": { "dataType": "float" }, "longhornVolumeNumberOfSnapshots": { "dataType": "float" }, "longhornVolumeReplicaAutoBalanceDisabledCount": { "dataType": "float" }, "longhornVolumeReplicaSoftAntiAffinityFalseCount": { "dataType": "float" }, "longhornVolumeReplicaZoneSoftAntiAffinityTrueCount": { "dataType": "float" }, "longhornVolumeReplicaDiskSoftAntiAffinityTrueCount": { "dataType": "float" }, "longhornVolumeRestoreVolumeRecurringJobFalseCount": { "dataType": "float" }, "longhornVolumeSnapshotDataIntegrityDisabledCount": { "dataType": "float" }, "longhornVolumeSnapshotDataIntegrityFastCheckCount": { "dataType": "float" }, "longhornVolumeUnmapMarkSnapChainRemovedFalseCount": { "dataType": "float" }, "longhornSettingOrphanResourceAutoDeletionGracePeriod": { "dataType": "float" }, "longhornSettingSnapshotHeavyTaskConcurrentLimit": { "dataType": "float" } } } image: repository: ${UPGRADE_RESPONDER_IMAGE_REPO} tag: ${UPGRADE_RESPONDER_IMAGE_TAG} EOF git clone -b ${UPGRADE_RESPONDER_REPO_BRANCH} ${UPGRADE_RESPONDER_REPO} helm upgrade --install ${APP_NAME}-upgrade-responder upgrade-responder/chart -f ${UPGRADE_RESPONDER_VALUE_YAML} wait_for_deployment "${APP_NAME}-upgrade-responder" } output() { local upgrade_responder_service_info=$(kubectl get svc/${APP_NAME}-upgrade-responder --no-headers) local upgrade_responder_service_port=$(echo "${upgrade_responder_service_info}" | awk '{print $5}' | cut -d'/' -f1) echo # a blank line to separate the installation outputs for better readability. printf "[Upgrade Checker]\n" printf "%-10s: http://${APP_NAME}-upgrade-responder.default.svc.cluster.local:${upgrade_responder_service_port}/v1/checkupgrade\n\n" "URL" printf "[InfluxDB]\n" printf "%-10s: ${INFLUXDB_URL}\n" "URL" printf "%-10s: ${APP_NAME}_upgrade_responder\n" "Database" printf "%-10s: root\n" "Username" printf "%-10s: root\n\n" "Password" local public_ip=$(curl -s https://ifconfig.me/ip) local grafana_service_info=$(kubectl get svc/grafana --no-headers) local grafana_service_port=$(echo "${grafana_service_info}" | awk '{print $5}' | cut -d':' -f2 | cut -d'/' -f1) printf "[Grafana]\n" printf "%-10s: http://${public_ip}:${grafana_service_port}\n" "Dashboard" printf "%-10s: admin\n" "Username" printf "%-10s: admin\n" "Password" } install_influxdb install_upgrade_responder install_grafana output ================================================ FILE: dev/upgrade-responder/manifests/grafana.yaml ================================================ --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: grafana-pvc spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 2Gi --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: grafana name: grafana spec: selector: matchLabels: app: grafana template: metadata: labels: app: grafana spec: securityContext: fsGroup: 472 supplementalGroups: - 0 containers: - name: grafana image: grafana/grafana:7.1.0 imagePullPolicy: IfNotPresent env: - name: GF_INSTALL_PLUGINS value: "grafana-worldmap-panel" ports: - containerPort: 3000 name: http-grafana protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /robots.txt port: 3000 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 30 successThreshold: 1 timeoutSeconds: 2 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 tcpSocket: port: 3000 timeoutSeconds: 1 resources: requests: cpu: 250m memory: 750Mi volumeMounts: - mountPath: /var/lib/grafana name: grafana-pv volumes: - name: grafana-pv persistentVolumeClaim: claimName: grafana-pvc --- apiVersion: v1 kind: Service metadata: name: grafana spec: ports: - port: 3000 protocol: TCP targetPort: http-grafana selector: app: grafana sessionAffinity: None type: LoadBalancer ================================================ FILE: dev/upgrade-responder/manifests/influxdb.yaml ================================================ apiVersion: v1 kind: Secret metadata: name: influxdb-creds namespace: default type: Opaque data: INFLUXDB_HOST: aW5mbHV4ZGI= # influxdb INFLUXDB_PASSWORD: cm9vdA== # root INFLUXDB_USERNAME: cm9vdA== # root --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: influxdb namespace: default labels: app: influxdb spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 2Gi --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: influxdb name: influxdb namespace: default spec: progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app: influxdb strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: creationTimestamp: null labels: app: influxdb spec: containers: - image: docker.io/influxdb:1.8.10 imagePullPolicy: IfNotPresent name: influxdb resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File envFrom: - secretRef: name: influxdb-creds volumeMounts: - mountPath: /var/lib/influxdb name: var-lib-influxdb volumes: - name: var-lib-influxdb persistentVolumeClaim: claimName: influxdb dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 30 --- apiVersion: v1 kind: Service metadata: labels: app: influxdb name: influxdb namespace: default spec: ports: - port: 8086 protocol: TCP targetPort: 8086 selector: app: influxdb sessionAffinity: None type: ClusterIP ================================================ FILE: enhancements/20200319-default-disks-and-node-configuration.md ================================================ # Default disks and node configuration ## Summary This enhancement allows the user to customize the default disks and node configurations in Longhorn for newly added nodes using Kubernetes label and annotation, instead of using Longhorn API or UI. ### Related Issues https://github.com/longhorn/longhorn/issues/1053 https://github.com/longhorn/longhorn/issues/991 ## Motivation ### Goals 1. Allow users to customize the disks and node configuration for new nodes without using Longhorn API or UI. This will make it much easier for users to scale the cluster since it will eliminate the necessity to configure Longhorn manually for each newly added node if the node contains more than one disk or the disk configuration is different between the nodes. 2. Allow users to define node tags for newly added nodes without using the Longhorn API or UI. ### Non-goals This enhancement will not keep the node label/annotation in sync with the Longhorn node/disks configuration. ## Proposal 1. Longhorn directly uses the node annotation to set the node tags once the node contains no tag. 2. Longhorn uses the setting `Create Default Disk on Labeled Nodes` to decide if to enable the default disks customization. If the setting is enabled, Longhorn will wait for the default disks customization set, instead of directly creating the Longhorn default disk for the node without disks (new node is included). Then Longhorn relies on the value of the node label `node.longhorn.io/create-default-disk` to decide how to customize default disks: If the value is `config`, the annotation will be parsed and used as the default disks customization. If the value is boolean value `true`, the data path setting will be used for the default disk. And other values will be treated as `false` and no default disk will be created. ### User Stories #### Scale up the cluster and add tags to new nodes Before the enhancement, when the users want to scale up the Kubernetes cluster and add tags on the node, they would need access to the Longhorn API/UI to do that. After the enhancement, the users can add a specified annotation to the new nodes to define the tags. In this way, the users don't need to work with Longhorn API/UI directly during the process of scaling up a cluster. #### Scale up the cluster and add disks to new nodes Before the enhancement, when the users want to scale up the Kubernetes cluster and customize the disks on the node, they would need to: 1. Enable the Longhorn setting `Create Default Disk on Labeled Nodes` to prevent the default disk to be created automatically on the node. 2. Add new nodes to the Kubernetes cluster, e.g. by using Rancher or Terraform, etc. 3. After the new node was recognized by Longhorn, edit the node to add disks using either Longhorn UI or API. The third step here needs to be done for every node separately, making it inconvenient for the operation. After the enhancement, the steps the user would take is: 1. Enable the Longhorn setting `Create Default Disk on Labeled Nodes`. 2. Add new nodes to the Kubernetes cluster, e.g. by using Rancher or Terraform, etc. 3. Add the label and annotations to the node to define the default disk(s) for the new node. Longhorn will pick it up automatically and add the disk(s) for the new node. In this way, the user doesn't need to work with Longhorn API/UI directly during the process of scaling up a cluster. ### User experience description #### Scenario 1 - Setup the default node tags: 1. The user adds the default node tags annotation `node.longhorn.io/default-node-tags=` to a Kubernetes node. 2. If the Longhorn node tag list was empty before step 1, the user should see the tag list for that node updated according to the annotation. Otherwise, the user should see no change to the tag list. #### Scenario 2 - Setup and use the default disks for a new node: 1. The users enable the setting `Create Default Disk on Labeled Nodes`. 2. The users add a new node, then they will get a node without any disk. 1. By deleting all disks on an existing node, the users can get the same result. 3. After patching the label `node.longhorn.io/create-default-disk=config` and the annotation `node.longhorn.io/default-disks-config=` for the Kubernetes node, the node disks should be updated according to the annotation. ## Design ### Implementation Overview ##### For Node Tags: If: 1. The Longhorn node contains no tag. 2. The Kubernetes node object of the same name contains an annotation `node.longhorn.io/default-node-tags`, for example: ``` node.longhorn.io/default-node-tags: '["fast","storage"]' ``` 3. The annotation can be parsed successfully. Then Longhorn will update the Longhorn node object with the new tags specified by the annotation. The process will be done as a part of the node controller reconciliation logic in the Longhorn manager. ##### For Default Disks: If: 1. The Longhorn node contains no disk. 2. The setting `Create Default Disk on Labeled Nodes` is enabled. 3. The Kubernetes node object of the same name contains the label `node.longhorn.io/create-default-disk: 'config'` and an annotation `node.longhorn.io/default-disks-config`, for example: ``` node.longhorn.io/default-disks-config: '[{"path":"/mnt/disk1","allowScheduling":false}, {"path":"/mnt/disk2","allowScheduling":false,"storageReserved":1024,"tags":["ssd","fast"]}]' ``` 4. The annotation can be parsed successfully. Then Longhorn will create the customized default disk(s) specified by the annotation. The process will be done as a part of the node controller reconciliation logic in the Longhorn manager. ##### Notice If the label/annotations failed validation, no partial configuration will be applied and the whole annotation will be ignored. No change will be done for the node tag/disks. The validation failure can be caused by: 1. The annotation format is invalid and cannot be parsed to tags/disks configuration. 2. The format is valid but there is an unqualified tag in the tag list. 3. The format is valid but there is an invalid disk parameter in the disk list. e.g., duplicate disk path, non-existing disk path, multiple disks with the same file system, the reserved storage size being out of range... ### Test plan 1. The users deploy Longhorn system. 2. The users enable the setting `Create Default Disk on Labeled Nodes`. 3. The users scale the cluster. Then the newly introduced nodes should contain no disk and no tag. 4. The users pick up a new node, create 2 random data path in the container then patch the following valid node label and annotations: ``` labels: node.longhorn.io/create-default-disk: "config" }, annotations: node.longhorn.io/default-disks-config: '[{"path":"","allowScheduling":false}, {"path":"","allowScheduling":true,"storageReserved":1024,"tags":["ssd","fast"]}]' node.longhorn.io/default-node-tags: '["fast","storage"]' ``` After the patching, the node disks and tags will be created and match the annotations. 5. The users use Longhorn UI to modify the node configuration. They will find that the node annotations keep unchanged and don't match the current node tag/disk configuration. 6. The users delete all node tags and disks via UI. Then the node tags/disks will be recreated immediately and match the annotations. 7. The users pick up another new node, directly patch the following invalid node label and annotations: ``` labels: node.longhorn.io/create-default-disk: "config" }, annotations: node.longhorn.io/default-disks-config: '[{"path":"","allowScheduling":false}, node.longhorn.io/default-node-tags: '["slow",".*invalid-tag"]' ``` Then they should find that the tag and disk list are still empty. 8. The users create a random data path then correct the annotation for the node: ``` annotations: node.longhorn.io/default-disks-config: '[{"path":"","allowScheduling":false}, node.longhorn.io/default-node-tags: '["slow","storage"]' ``` Now they will see that the node tags and disks are created correctly and match the annotations. ### Upgrade strategy N/A. ================================================ FILE: enhancements/20200331-replace-filesystem-id-key-in-disk-map.md ================================================ # Replace Filesystem ID key in Disk map ## Summary This enhancement will remove the dependency of filesystem ID in the DiskStatus, because we found there is no guarantee that filesystem ID won't change after the node reboots for e.g. XFS. ### Related Issues https://github.com/longhorn/longhorn/issues/972 ## Motivation ### Goals 1. Previously Longhorn is using filesystem ID as keys to the map of disks on the node. But we found there is no guarantee that filesystem ID won't change after the node reboots for certain filesystems e.g. XFS. 1. We want to enable the ability to configure CRD directly, prepare for the CRD based API access in the future 1. We also need to make sure previously implemented safe guards are not impacted by this change: 1. If a disk was accidentally unmounted on the node, we should detect that and stop replica from scheduling into it. 1. We shouldn't allow user to add two disks pointed to the same filesystem ### Non-goals For this enhancement, we will not proactively stop replica from starting if the disk it resides in is NotReady. Lack of `replicas` directory should stop replica from starting automatically. ## Proposal We will generate UUID for each disk called `diskUUID` and store it as a file `longhorn-disk.cfg` in the filesystem on the disk. If the filesystem already has the `diskUUID` stored, we will retrieve and verify the `diskUUID` and make sure it doesn't change when we scan the disks. The disk name can be customized by user as well. ### Background Filesystem ID was a good identifier for the disk: 1. Different filesystems on the same node will have different filesystem IDs. 1. It's built-in in the filesystem. Only need one command(`stat`) to retrieve it. But there is another assumption we had which turned out not to be true. We assumed filesystem ID won't change during the lifecycle of the filesystem. But we found that some filesystem ID can change after a remount. It caused an issue on XFS. Besides that, there is another problem we want to address: currently API server is forwarding the request of updateDisks to the node of the disks, since only that node has access to the filesystem so it can fill in the FilesystemID(`fsid`). As long as we're using the `fsid` as the key of the disk map, we cannot create new disks without let the node handling the request. This become an issue when we want to allow direct editing CRDs as API. ### User Experience In Detail Before the enhancement, if the users add more disks to the node, API gateway will forward the request to the responsible node, which will validate the input on the fly for cases like two disks point to the same filesystem. After the enhancement, when the users add more disks to the node, API gateway will only validate the basic input. The other error cases will be reflected in the disk's Condition field. 1. If different disks point to the same directory, then: 1. If all the disks are added new, both disks will get condition `ready = false`, with the message indicating that they're pointing to the same filesystem. 1. If one of the disks already exists, the other disks will get condition `ready = false`, with the message indicating that they're pointing to the same filesystem as one of the existing disks. 1. If there is more than one disk exists and pointing to the same filesystem. Longhorn will identify which disk is the valid one using `diskUUID` and set the condition of other disks to `ready = false`. ### API changes 1. API input for the diskUpdate call will be a map[string]DiskSpec instead of []DiskSpec. 1. API no longer validates duplicate filesystem ID. ### UI changes UI can let the user customize the disk name. By default UI can generate name like `disk-` for the disks. ## Design ### Implementation Overview The validation of will be done in the node controller `syncDiskStatus`. syncDiskStatus process: 1. Scan through the disks, and record disks in the FSID to disk map 1. Check for each FSID after the scanning is done. 1. If there is only one disk in for a FSID 1. If the disk already has `status.diskUUID` 1. Check for file `longhorn-disk.cfg` 1. file exists: parse the value. If it doesn't match status.diskUUID, mark the disk as NotReady 1. case: mount the wrong disk. 1. file doesn't exist: mark the disk as NotReady 1. case: Reboot and forget to mount. 1. If the disk has empty `status.diskUUID` 1. check for file `longhorn-disk.cfg`. 1. if exists, parse uuid. 1. If there is no duplicate UUID in the disk list, then record the uuid 1. Otherwise mark as NotReady `duplicate UUID`. 1. if not exists, generate the uuid, record it in the file, then fill in `status.diskUUID`. 1. Creating new disk. 1. If there are more than one disks with the same FSID 1. if the disk has `status.diskUUID` 1. follow 2.i.a 1. If the disk doesn't have `status.diskUUID` 1. mark as NotReady due to duplicate FSID. #### Note on the disk naming The default disks of the node will be called `default-disk-`. That includes the default disks created using node labels/annotations. ### Test plan Update existing test plan on node testing will be enough for the first step, since it's already covered the case for changing filesystem. ### Upgrade strategy No change for previous disks since they all used the FSID which is at least unique on the node. Node controller will fill `diskUUID` field and create `longhorn-disk.cfg` automatically on the disk once it processed it. ================================================ FILE: enhancements/20200625-volume-deletion-flows.md ================================================ # Volume Deletion Flows ## Summary This enhancement modifies the flow a user would follow for handling deletion of `Volumes` that are `Attached` or otherwise have resources such as a `Persistent Volume` associated with them. Specifically, this adds warnings in the `longhorn-ui` when deleting a `Volume` in these specific cases and provides a means for the `longhorn-manager` to clean up any leftover resources in `Kubernetes` associated with a deleted `Volume`. ### Related Issues https://github.com/longhorn/longhorn/issues/520 ## Motivation ### Goals The goal of this enhancement is to either address or warn users about situations in which deleting a `Volume` could cause potential problems. In handling the case of cleaning up an associated `Persistent Volume` (and possibly `Persistent Volume Claim`), we can prevent there being leftover unusable `Volume`-related resources in `Kubernetes`. In warning about deletion when the `Volume` is attached, we can inform the user about possible consequences the deletion would have on existing workloads so the user can handle this accordingly. ### Non-Goals This enhancement is not intended on completely blocking a user from pursuing any dangerous operations. For example, if a user insists on deleting a currently attached `Volume`, they should not be forbidden from doing so in case the user is absolutely sure that they want to follow through. ## Proposal When a user wishes to delete a `Volume` from the `Longhorn UI`, the system should check to see if the `Volume` has a resource tied to it or is currently `Attached`: - If the `Volume` is `Attached`, the user should be warned about the potential consequences of deleting the `Volume` (namely that any applications currently using the `Volume` will no longer have access to it and likely error out) before they can confirm the deletion or cancel it. - If the `Volume` is tied to a `Volume` that is tied to a `Persistent Volume` (and possibly a `Persistent Volume Claim`), the user should be informed of this information and the fact that we will clean up those resources if the `Volume` is deleted. If the `Volume` is tied to a `Persistent Volume Claim`, the user should also be warned that there may be `Deployments` or `Stateful Sets` that depend on this `Volume` that could no longer work should the user delete the `Volume` (we cannot explicitly see this without having to monitor all `Deployments` and `Stateful Sets` to check if they use a `Longhorn`-backed `Persistent Volume Claim`). Afterwards, the user can confirm the deletion if they wish, which will lead to cleanup of the associated resources and deletion of the `Volume`. ### User Stories #### Deletion of Volumes with Associated Resources Before, a user deleting a `Volume` through the `longhorn-ui` would only face the default confirmation message. The user would see the related `Persistent Volume` (and possibly `Persistent Volume Claim`) from the `Volume` listing, but this information would not be displayed in the confirmation message, and on deletion, these resources would still exist, which could raise problems if a user attempted to use these in a workload since they would refer to a nonexistent `Volume`. After this enhancement, the user would be alerted about the existence of these resources and the fact that deletion of the `Volume` would lead to cleanup of these resources. The user can decide as normal whether to follow through with deletion of the `Volume` from the `longhorn-ui` or not. #### Deletion of an Attached Volume Before, a user deleting a `Volume` that was `Attached` would only face the default confirmation message in the `longhorn-ui`. The fact that the `Volume` was `Attached` would not be indicated in the confirmation message, and the user could potentially cause errors in applications using the `Volume` without any warnings. After this enhancement, a user would be alerted about the `Volume` being `Attached` and would be able to decide on a course of action for `Volume` deletion and handling of any applications using the `Volume` accordingly. ### User Experience In Detail #### Deletion of Volumes with Associated Resources 1. The user attempts to delete a `Volume` that has a `Persistent Volume` (and potentially a `Persistent Volume Claim`) associated with it. 2. The confirmation message will appear, asking the user to confirm the operation. Additionally, the message will tell the user that the `longhorn-manager` will delete the `Kubernetes` resources associated with the `Volume`. If the `Volume` is additionally tied to a `Persistent Volume Claim`, the user will also be warned about possible adverse effects for any `Deployments` or `Stateful Sets` that may be using that `Volume`. 3. The user can now follow through with one of two options: - They can press `Cancel`, which will do nothing and take them back to the `Volume` listing. - They can press `Confirm` to follow through with the operation. The `longhorn-manager` will process deletion of the `Volume` and automatically clean up any associated `Persistent Volume` or `Persistent Volume Claim`. #### Deletion of an Attached Volume 1. The user attempts to delete a `Volume` from the `longhorn-ui` that is currently `Attached`. 2. The confirmation message will appear, telling the user that the `Volume` is `Attached` and that deleting the `Volume` can lead to errors in any applications using the `Volume`. - If the `Volume` is also attached to a `Kubernetes` workload (we can determine this from the `Kubernetes Status`) 3. The user can now follow through with one of two options: - They can press `Cancel`, which will do nothing and take them back to the `Volume` listing. - They can press `Confirm` to follow through with the operation. The `longhorn-manager` will process deletion of the `Volume`. The user will be responsible for handling any errored applications that depend on the now-deleted `Volume`. ### API Changes From an API perspective, the call made to delete the `Volume` should look the same. The logic for handling deletion of any `Persistent Volume` or `Persistent Volume Claim` should go into the `Volume Controller`. ## Design ### Implementation Overview 1. `longhorn-ui` changes: - When a user attempts to delete a `Volume`: - If the `Volume` has an associated `Persistent Volume` and possibly `Persistent Volume Claim`, add an additional warning to the confirmation dialog regarding cleanup of these resources. - If the `Volume` is `Attached`, add an additional warning to the confirmation dialog regarding possible errors that may occur that the user should account for. 2. `longhorn-manager` changes: - In the `Volume Controller`, if a `Volume` has a `Deletion Timestamp`, check the `Kubernetes Status` of the `Volume`: - If there is a `Persistent Volume`, delete it. - If there is a `Persistent Volume Claim`, delete it. ### Test Plan A number of integration tests will need to be added for the `longhorn-manager` in order to test the changes in this proposal: 1. From the API, create a `Volume` and then create a `Persistent Volume` and `Persistent Volume Claim`. Wait for the `Kubernetes Status` to be populated. Attempt to delete the `Volume`. Both the `Persistent Volume` and `Persistent Volume Claim` should be deleted as well. 2. Create a `Storage Class` for `Longhorn` and use that to provision a new `Volume` for a `Persistent Volume Claim`. Attempt to delete the `Volume`. Both the `Persistent Volume` and `Persistent Volume Claim` should be deleted as well. Additionally, some manual testing will need to be performed against the `longhorn-ui` changes for this proposal: 1. From the `longhorn-ui`, create a new `Volume` and then create a `Persistent Volume` for that `Volume`. Attempt to delete the `Volume`. The dialog box should indicate the user that there will be `Kubernetes` resources that will be deleted as a result. 2. From the `longhorn-ui`, create a new `Volume` and then `Attach` it. Attempt to delete the `Volume`. The dialog box should indicate that the `Volume` is in use and warn about potential errors. 3. Use `Kubernetes` to create a `Volume` and use it in a `Pod`. Attempt to delete the `Volume` from the `longhorn-ui`. Multiple warnings should show up in the dialog box, with one indicating removal of the `Kubernetes` resources and the other warning about the `Volume` being in use. ### Upgrade strategy No special upgrade strategy is necessary. Once the user upgrades to the new version of `Longhorn`, these new capabilities will be accessible from the `longhorn-ui` without any special work. ### Notes - There is interest in allowing the user to decide on whether or not to retain the `Persistent Volume` (and possibly `Persistent Volume Claim`) for certain use cases such as restoring from a `Backup`. However, this would require changes to the way `go-rancher` generates the `Go` client that we use so that `Delete` requests against resources are able to take inputs. - In the case that a `Volume` is provisioned from a `Storage Class` (and set to be `Deleted` once the `Persistent Volume Claim` utilizing that `Volume` has been deleted), the `Volume` should still be deleted properly regardless of how the deletion was initiated. If the `Volume` is deleted from the UI, the call that the `Volume Controller` makes to delete the `Persistent Volume` would only trigger one more deletion call from the `CSI` server to delete the `Volume`, which would return successfully and allow the `Persistent Volume` to be deleted and the `Volume` to be deleted as well. If the `Volume` is deleted because of the `Persistent Volume Claim`, the `CSI` server would be able to successfully make a `Volume` deletion call before deleting the `Persistent Volume`. The `Volume Controller` would have no additional resources to delete and be able to finish deletion of the `Volume`. ================================================ FILE: enhancements/20200701-backupstore-file-locks.md ================================================ # Backupstore File Locks ## Summary This enhancement will address backup issues that are the result of concurrently running backup operations, by implementing a synchronisation solution that utilizes files on the backup store as Locks. ### Related Issues https://github.com/longhorn/longhorn/issues/612 https://github.com/longhorn/longhorn/issues/1393 https://github.com/longhorn/longhorn/issues/1392 https://github.com/longhorn/backupstore/pull/37 ## Motivation ### Goals Identify and prevent backup issues caused as a result of concurrent backup operations. Since it should be safe to do backup creation & backup restoration at the same time, we should allow these concurrent operations. ## Proposal The idea is to implement a locking mechanism that utilizes the backupstore, to prevent the following dangerous cases of concurrent operations. 1. prevent backup deletion during backup restoration 2. prevent backup deletion while a backup is in progress 3. prevent backup creation during backup deletion 4. prevent backup restoration during backup deletion The locking solution shouldn't unnecessary block operations, so the following cases should be allowed. 1. allow backup creation during restoration 2. allow backup restoration during creation The locking solution should have a maximum wait time for lock acquisition, which will fail the backup operation so that the user does not have to wait forever. The locking solution should be self expiring, so that when a process dies unexpectedly, future processes are able to acquire the lock. The locking solution should guarantee that only a single type of lock is active at a time. The locking solution should allow a lock to be passed down into async running go routines. ### User Experience In Detail Before this enhancement, it is possible to delete a backup while a backup restoration is in progress. This would lead to an unhealthy restoration volume. After this enhancement, a backup deletion could only happen after the restoration has been completed. This way the backupstore continues to contain all the necessary blocks that are required for the restoration. After this enhancement, creation & restoration operations are mutually exclusive with backup deletion operations. ### API changes ## Design ### Implementation Overview Conceptually the lock can be thought of as a **RW** lock, it includes a `Type` specifier where different types are mutually exclusive. To allow the lock to be passed into async running go routines, we add a `count` field, that keeps track of the current references to this lock. ```go type FileLock struct { Name string Type LockType Acquired bool driver BackupStoreDriver volume string count int32 serverTime time.Time refreshTimer *time.Ticker } ``` To make the lock self expiring, we rely on `serverTime` updates which needs to be refreshed by a timer. We chose a `LOCK_REFRESH_INTERVAL` of **60** seconds, each refresh cycle a locks `serverTime` will be updated. A lock is considered expired once the current time is after a locks `serverTime` + `LOCK_MAX_WAIT_TIME` of **150** seconds. Once a lock is expired any currently active attempts to acquire that lock will timeout. ```go const ( LOCKS_DIRECTORY = "locks" LOCK_PREFIX = "lock" LOCK_SUFFIX = ".lck" LOCK_REFRESH_INTERVAL = time.Second * 60 LOCK_MAX_WAIT_TIME = time.Second * 150 LOCK_CHECK_INTERVAL = time.Second * 10 LOCK_CHECK_WAIT_TIME = time.Second * 2 ) ``` Lock Usage 1. create a new lock instance via `lock := lock.New()` 2. call `lock.Lock()` which will block till the lock has been acquired and increment the lock reference count. 3. defer `lock.Unlock()` which will decrement the lock reference count and remove the lock once unreferenced. To make sure the locks are **mutually exclusive**, we use the following process to acquire a lock. 1. create a lock file on the backupstore with a unique `Name`. 2. retrieve all lock files from the backupstore order them by `Acquired` then by `serverTime` followed by `Name` 3. check if we can acquire the lock, we can only acquire if there is no unexpired(i) lock of a different type(ii) that has priority(iii). 1. Locks are self expiring, once the current time is after `lock.serverTime + LOCK_MAX_WAIT_TIME` we no longer need to consider this lock as valid. 2. Backup & Restore Locks are mapped to compatible types while Delete Locks are mapped to a different type to be mutually exclusive with the others. 3. Priority is based on the comparison order, where locks are compared by `lock.Acquired` then by `lock.serverTime` followed by `lock.Name`. Where acquired locks are always sorted before non acquired locks. 4. if lock acquisition times out, return err which will fail the backup operation. 5. once the lock is acquired, continuously refresh the lock (updates `lock.serverTime`) 5. once the lock is acquired, it can be passed around by calling `lock.Lock()` 6. once the lock is no longer referenced, it will be removed from the backupstore. It's very unlikely to run into lock collisions, since we use uniquely generated name for the lock filename. In cases where two locks have the same `lock.serverTime`, we can rely on the `lock.Name` as a differentiator between 2 locks. ### Test plan A number of integration tests will need to be added for the `longhorn-engine` in order to test the changes in this proposal: 1. place an expired lock file into a backupstore, then verify that a new lock can be acquired. 2. place an active lock file of Type `Delete` into a backupstore, then verify that backup/restore operations will trigger lock acquisition timeout. 3. place an active lock file of Type `Delete` into a backupstore, then verify that a new `Delete` operation can acquire a lock. 4. place an active lock file of Type `Backup/Restore` into a backupstore, then verify that delete operations will trigger lock acquisition timeout. 5. place an active lock file of Type `Backup/Restore` into a backupstore, then verify that a new `Backup/Restore` operation can acquire a lock. ### Upgrade strategy No special upgrade strategy is necessary. ================================================ FILE: enhancements/20200721-refactor-restore-for-rebuild-enabling.md ================================================ # Refactor restore for rebuild enabling ## Summary This enhancement will refactor the restore implementation and enable rebuild for restore/DR volumes. ### Related Issues https://github.com/longhorn/longhorn/issues/1279 ## Motivation ### Goals The goal of this enhancement is to simplify the restore flow so that it can work for rebuilding replicas of restore/DR volumes without breaking the live upgrade feature. ### Non-goals This enhancement won't guarantee that the restore/DR volume activation won't be blocked by replica rebuilding. ## Proposal - When there are replicas crashing among restore/DR volumes, new rebuilding replicas will be created as usual. But instead of following the normal replica rebuilding workflow (syncing data/files from other running replicas), the rebuilding replicas of restore/DR volumes will directly restore data from backup. - The normal rebuilding (file syncing) workflow implicitly considers that all existing snapshots won't change and newer data during the rebuilding will be written into volume head. But for restore/DR volumes, new data writing is directly handled by replica (sync agent server) and it will be written to underlying snapshots rather than volume heads. As a result, the normal rebuilding logic doesn't fit restore/DR volumes. - In order to skip the file syncing and snapshotting and directly do restore, the rebuilding related API should be updated, which will lead to API version bumps. - As long as there is a replica having not restored the newest/latest backup, longhorn manager will directly call restore command. Then rebuilding replicas will be able to start the restore even if all other replicas are up-to-date. - Previously, in order to maintain the consistency of DR volume replicas, Longhorn manager will guarantee that all replicas have restored the same backup before starting the next backup restore. But considering the case that the newly rebuilt replicas are empty whereas the existing replicas have restored some backups, This restriction makes replica rebuilding become impossible in some cases. Hence we need to break the restriction. - This restriction break degrades the consistency of DR volume replicas. But it's acceptable as long as all replicas can finish the latest backup restore and the DR volume can be activated in the end. - This modification means engines and replicas should be intelligent enough to decide if they need to do restore and which kind of restore they need to launch. - Actually replica processes have all information about the restore status and they can decide if they need incremental restore or full restore by themself. Specifying the last backup in the restore command is redundant. - Longhorn manager only needs to tell the replicas what is the latest backup they should restore. - Longhorn manager still need to know what is the last restored backup of all replicas, since it relies on it to determine if the restore/DR volume is available/can be activated. - Longhorn should wait for rebuild complete and check restore status before auto detachment. - Otherwise, the restore volume will be automatically detached when the rebuild is in progress then the rebuild is meaningless in this case. ### User Stories #### Replica crashes when a restore/DR volume is in restore progress Before, the restore volume keeps state `Degraded` if there is replica crashing. And the volume will finally become `Faulted` if all replicas are crashed one by one during restoring. After, the restore volume will start replica rebuilding automatically then be back to state `Healthy` if there is replica crashing. The volume is available as long as all replicas are not crashed at the same time. And volume will finish activation/auto-detachment after the rebuild is done. ### User Experience In Detail #### Replica crash on restore volume 1. Users create a restore volume and wait for restore complete. 2. When the restore is in progress, some replicas somehow get crashed. Then the volume rebuilds new replicas immediately, and it will become `Healthy` once the new replicas start rebuilding. 4. The volume will be detached automatically once the restore and the rebuild complete. #### Replica crash on DR volume 1. Users create a DR volume. 2. Some replicas get crashed. Then the DR volume automatically rebuilds new replicas and restores the latest backup for the rebuilt replicas. 3. Users try to activate the DR volume. The DR volume will wait for the rebuild of all replicas and successful restoration of the latest backup before detachment. ### API changes #### CLI API - Add a new flag `--restore` for command `add-replica`, which indicates skipping file syncing and snapshotting. - Deprecate the arg `lastRestoreBackup` and the flag `--incrementally` for command `backup restore`. - Add a new command `verify-rebuild-replica`, which can mark the rebuilding replicas as available (mode `RW`) for restore/DR volumes after the initial restore is done. #### Controller gRPC API - Create a separate message/struct for `ReplicaCreate` request then add the two new fields `Mode` and `SnapshotRequired` to the request. ## Design ### Implementation Overview #### Engine Part: 1. Modify command `add-replica` related APIs: 1. Use a new flag `--restore` in command `add-replica` to indicate that file syncing and snapshotting should be skipped for restore/DR volumes. 2. The current controller gRPC call `ReplicaCreate` used in the command will directly create a snapshot before the rebuilding. But considering the (snapshot) consistency of of restore/DR volumes, snapshots creation/deletion is fully controlled by the restore command (and the expansion command during the restore). Hence, the snapshotting here needs to be skipped by updating the gRPC call `ReplicaCreate`. 2. Add command `verify-rebuild-replica`: 1. It just calls the existing controller gRPC function `ReplicaVerifyRebuild`. 2. It's mainly used to mark the rebuilding replica of restore/DR volumes as mode `RW` with some verifications and a replica reload. 3. Modify command `backup restore`: 1. Deprecate/Ignore the arg `lastRestoreBackup` in the restore command and the following sync agent gRPC function. Instead, the sync agent server will directly do a full restore or a incremental restore based on its current restore status. 2. Deprecate/Ignore the flag `--incrementally` for command `backup restore`. By checking the disk list of all existing replicas, the command function knows if it needs to generate a new snapshot name. 3. The caller of the gRPC call `BackupRestore` only needs to tell the name of the final snapshot file that stores restored data. 1. For new restore volume, there is no existing snapshot among all replicas hence we will generate a random snapshot name. 2. For replicas of DR volumes or rebuilding replicas of restore volumes, the caller will find the replica containing the most snapshots then use the latest snapshot of the replica in the following restore. 3. As for the delta file used in the incremental restore, it will be generated by the sync agent server rather than by the caller. Since the caller has no idea about the last restored backup now and the delta file naming format is `volume-delta-.img`. 4. To avoid disk/snapshot chain inconsistency between rebuilt replicas and old replicas of a DR volume, snapshot purge is required if there are more than 1 snapshots in one replica. And the (incremental) restore will be blocked before the snapshot purge complete. 3. Make the sync agent gRPC call `BackupRestore` more “intelligent”: The function will check the restore status first. If there is no restore record in the sync agent server or the last restored backup is invalid, a full restore will be applied. This means we can remove the gRPC call `BackupRestoreIncrementally`. 4. Remove the expansion before the restore call. The expansion of DR volumes should be guaranteed by longhorn manager. 5. Coalesce the incremental restore related functions to normal restore functions if possible. #### Manager Part: 1. Allow replica replenishment for restore/DR volumes. 2. Add the new flag `--restore` when using command `add-replica` to rebuild replicas of restore/DR volumes. 3. Modify the pre-restore check and restore status sync logic: 1. Previously, the restore command will be invoked only if there is no restoring replica. Right now the command will be called as long as there is a replica having not restored the latest backup. 2. Do not apply the consensual check as the prerequisite of the restore command invocation. The consensual check will be used for `engine.Status.LastRestoredBackup` update only. 3. Invoke `verify-rebuild-replica` when there is a complete restore for a rebuilding replica (mode `WO`). 4. Modify the way to invoke restore command: 1. Retain the old implementation for compatibility. 2. For the engine using the new engine image, call restore command directly as long as the pre-restore check gets passed. 3. Need to ignore some errors. e.g.: replicas are restoring, the requested backup restore is the same as the last backup restore, or replicas need to complete the snapshot purge before the restore. 5. Mark the rebuilding replicas as mode `ERR` and disable the replica replenishment during the expansion. 6. Modify the prerequisites of restore volume auto detachment or DR volume activation: 1. Wait for the rebuild complete and the volume becoming `Healthy`. 2. Check and wait for the snapshot purge. 3. This prerequisite check works only for new restore/DR volumes. ### Test plan #### Engine integration tests: ##### Restore volume simple rebuild: 1. Create a restore volume with 2 replicas. 2. Run command `backup restore` for the DR volume. 3. Delete one replica of the restore volume. 4. Initialize a new replica, and add the replica to the restore volume. 5. Run command `backup restore`. 6. Verify the restored data is correct, and all replicas work fine. ##### DR volume rebuild after expansion: 1. Create a DR volume with 2 replicas. 2. Run command `backup restore` for the DR volume. 3. Wait for restore complete. 4. Expand the DR volume and wait for the expansion complete. 5. Delete one replica of the DR volume. 6. Initialize a new replica, and add the replica to the DR volume. 7. Run command `backup restore`. The old replica should start snapshot purge and the restore is actually not launched. 8. Wait for the snapshot purge complete. 9. Re-run command `backup restore`. Then wait for the restore complete. 10. Check if the restored data is correct, and all replicas work fine. And verify all replicas contain only 1 snapshot. #### Manager integration tests: ##### Restore volume rebuild: 1. Launch a pod with Longhorn volume. 2. Write data to the volume and take a backup. 3. Create a restore volume from the backup and wait for the restore start. 4. Crash one random replicas. Then check if the replicas will be rebuilt and the restore volume can be `Healthy` after the rebuilding. 5. Wait for the restore complete and auto detachment. 6. Launch a pod for the restored volume. 7. Verify all replicas work fine with the correct data. ##### DR volume rebuild during the restore: 1. Launch a pod with Longhorn volume. 2. Write data to the volume and take the 1st backup. 3. Wait for the 1st backup creation complete then write more data to the volume (which is the data of the 2nd backup). 4. Create a DR volume from the 1st backup and wait for the restore start. 5. Crash one random replica. 6. Take the 2nd backup for the original volume. Then trigger DR volume last backup update immediately (by calling backup list API) after the 2nd backup creation complete. 7. Check if the replicas will be rebuilt and the restore volume can be `Healthy` after the rebuilding. 8. Wait for the restore complete then activate the volume. 9. Launch a pod for the activated DR volume. 10. Verify all replicas work fine with the correct data. ##### DR volume rebuild with expansion: 1. Launch a pod with Longhorn volume. 2. Write data to the volume and take the 1st backup. 3. Create a DR volume from the 1st backup. 4. Shutdown the pod and wait for the original volume detached. 5. Expand the original volume and wait for the expansion complete. 6. Re-launch a pod for the original volume. 7. Write data to the original volume and take the 2nd backup. (Make sure the total data size is larger than the original volume size so that there is date written to the expanded part.) 8. Wait for the 2nd backup creation complete. 9. Trigger DR volume and crash one random replica of the DR volume. 10. Check if the replicas will be rebuilt, and the restore volume can be `Healthy` after the rebuilding. 11. Wait for the expansion, restore, and rebuild complete. 12. Verify the DR volume size and snapshots count after the restore. 13. Write data to the original volume and take the 3rd backup. 14. Wait for the 3rd backup creation complete then trigger the incremental restore for the DR volume. 15. Activate the DR volume and wait for the DR volume activated. 16. Launch a pod for the activated DR volume. 17. Verify the restored data of the activated DR volume. 18. Write more data to the activated DR volume. Then verify all replicas are still running. 19. Crash one random replica of the activated DR volume. 20. Wait for the rebuild complete then verify the activated volume still works fine. ### Manual test 1. Launch Longhorn v1.0.1. 2. Launch a pod with Longhorn volume. 3. Write data to the volume and take the 1st backup. 4. Create 2 DR volumes from the 1st backup. 5. Shutdown the pod and wait for the original volume detached. 6. Expand the original volume and wait for the expansion complete. 7. Write data to the original volume and take the 2nd backup. (Make sure the total data size is larger than the original volume size so that there is date written to the expanded part.) 8. Trigger incremental restore for the DR volumes by listing the backup volumes, and wait for restore complete. 9. Upgrade Longhorn to the latest version. 10. Crash one random replica for the 1st DR volume . 11. Verify the 1st DR volume won't rebuild replicas and keep state `Degraded`. 12. Write data to the original volume and take the 3rd backup. 13. Trigger incremental restore for the DR volumes, and wait for restore complete. 14. Do live upgrade for the 1st DR volume. This live upgrade call should fail and nothing gets changed. 15. Activate the 1st DR volume. 16. Launch a pod for the 1st activated volume, and verify the restored data is correct. 17. Do live upgrade for the original volume and the 2nd DR volumes. 18. Crash one random replica for the 2nd DR volume. 19. Wait for the restore & rebuild complete. 20. Delete one replica for the 2nd DR volume, then activate the DR volume before the rebuild complete. 21. Verify the DR volume will be auto detached after the rebuild complete. 22. Launch a pod for the 2nd activated volume, and verify the restored data is correct. 23. Crash one replica for the 2nd activated volume. 24. Wait for the rebuild complete, then verify the volume still works fine by reading/writing more data. ### Upgrade strategy Live upgrade is supported. ## Note It's possible that the restore/DR volume rebuilding somehow gets stuck, or users have no time to wait for the restore/DR volume rebuilding done. We need to provide a way that users can use the volume as soon as possible. This enhancement is tracked in https://github.com/longhorn/longhorn/issues/1512. ================================================ FILE: enhancements/20200727-add-replica-eviction-support-for-disks-and-nodes.md ================================================ # Replica Eviction Support for Disks and Nodes ## Summary This enhancement is to simplify and automatically evict the replicas on the selected disabled disks or nodes to other suitable disks and nodes per user's request. Meanwhile keep the same level of fault tolerance during this eviction period of time. ### Related Issues https://github.com/longhorn/longhorn/issues/292 https://github.com/longhorn/longhorn/issues/298 ## Motivation ### Goals 1. Allow user easily evict the replicas on the selected disks or nodes to other disks or nodes without impact the user defined `Volume.Spec.numberOfReplicas` and keep the same level of fault tolerance. This means we don't change the user defined replica number. 2. Report any error to user during the eviction time. 3. Allow user to cancel the eviction at any time. ## Proposal 1. Add `Eviction Requested` with `true` and `false` selection buttons for disks and nodes. This is for user to evict or cancel the eviction of the disks or the nodes. 2. Add new `evictionRequested` field to `Node.Spec`, `Node.Spec.disks` Spec and `Replica.Status`. These will help tracking the request from user and trigger replica controller to update `Replica.Status` and volume controller to do the eviction. And this will reconcile with `scheduledReplica` of selected disks on the nodes. 3. Display `fail to evict` error message to `Dashboard` and any other eviction errors to the `Event log`. ### User Stories ### Disks and Nodes Eviction For disk replacement or node replacement, the eviction needs to be done successfully in order to guarantee Longhorn volume function properly. Before, when user wants to evict a disk or a node they need to do the following steps: 1. User needs to disable the disk or the node. 2. User needs to scale up the replica count for the volume which has replica on disabled disks or nodes, and wait for the rebuild complete, scale down the replica count, then delete the replicas on this disk or node. After this enhancement, user can click `true` to the `Eviction Requested` on scheduling disabled disks or nodes. Or select `Disable` for scheduling and `true` to the `Eviction Requested` at the same time then save this change. The backend will take care of the eviction for the disks or nodes and cleanup for all the replicas on disks or nodes. ### User Experience In Detail #### Disks and Nodes Eviction 1. User can select `true` to the `Eviction Requested` from `Longhorn UI` for disks or nodes. And user has to make sure the selected disks or nodes have been disabled, or select the `Disable` Scheduling at the same time of `true` to the `Eviction Requested`. 2. Once `Eviction Requested` has been set to `true` on the disks or nodes, they can not be enabled for `Scheduling`. 3. If the disks or the nodes haven't been disabled for `Scheduling`, there will be error message showed in `Dashboard` immediately to indicate that user need to disable the disk or node for eviction. 4. And user will wait for the replica number for the disks or nodes to be 0. 5. If there is any error e.g. no space or couldn't find other schedulable disk, the error message will be logged in the `Event log`. And the eviction will be suspended until either user sets the `Eviction Requested` to `false` or cleanup more disk spaces for the new replicas. 6. If user cancel the eviction by setting the `Eviction Requested` to `false`, the remaining replicas on the selected disks or nodes will remain on the disks or nodes. ### API changes From an API perspective, the call to set `Eviction Requested` to `true` or `false` on the `Node` or `Disk` eviction should look the same. The logic for handling the new field `Eviction Requested` `true` or `false` should to be in the `Node Controller` and `Volume Controller`. ## Design ### Implementation Overview 1. On `Longhorn UI` `Node` page, for nodes eviction, adding `Eviction Requested` `true` and `false` options in the `Edit Node` sub-selection, next to `Node Scheduling`. For disks eviction, adding `Eviction Requested` `true` and `false` options in `Edit node and disks` sub-selection under `Operation` column next to each disk `Scheduling` options. This is for user to evict or cancel the eviction of the disks or the nodes. 2. Add new `evictionRequested` field to `Node.Spec`, `Node.Spec.disks` Spec and `Replica.Status`. These will help tracking the request from user and trigger replica controller to update `Replica.Status` and volume controller to do the eviction. And this will reconcile with `scheduledReplica` of selected disks on the nodes. 3. Add a informer in `Replica Controller` to get these information and update `evictionRequested` field in `Replica.Status`. 4. Once `Eviction Requested` has been set to `true` for disks or nodes, the `evictionRequested` fields for the disks and nodes will be set to `true` (default is `false`). 5. `Replica Controller` will update `evictionRequested` field in `Replica.Status` and `Volume Controller` to get these information from it's replicas. 6. During reconcile the engine replica, based on `Replica.Status.EvictionRequested` of the volume replicas to trigger rebuild for different volumes' replicas. And remove one replica with `evictionRequested` `true`. 7. Logged the errors to `Event log` during the reconcile process. 8. By the end from `Longhorn UI`, the replica number on the eviction disks or nodes should be 0, this mean eviction is success. 9. If the volume is 'Detached', Longhorn will 'Automatically Attach' the volume and do the eviction, after eviction success, the volume will be 'Automatically detach'. If there is any error during the eviction, it will get suspended, until user solve the problem, the 'Auto Detach' will be triggered at the end. ### Test plan #### Manual Test Plan For Disks and Nodes Eviction Positive Case: For both `Replica Node Level Soft Anti-Affinity` has been enabled and disabled. Also the volume can be 'Attached' or 'Detached'. 1. User can select one or more disks or nodes for eviction. Select `Eviction Requested` to `true` on the disabled disks or nodes, Longhorn should start rebuild replicas for the volumes which have replicas on the eviction disks or nodes, and after rebuild success, the replica number on the evicted disks or nodes should be 0. E.g. When there are 3 nodes in the cluster, and with `Replica Node Level Soft Anti-Affinity` is set to `false`, disable one node, and create a volume with replica count 2. And then evict one of them, the eviction should get stuck, then set `Replica Node Level Soft Anti-Affinity` to `true`, the eviction should go through. Negative Cases: 1. If user selects the disks or nodes have not been disabled scheduling, Longhorn should display the error message on `Dashboard` immediately. Or during the eviction, the disabled disk or node can not be re-enabled again. 2. If there is no enough disk spaces or nodes for disks or nodes eviction, Longhorn should log the error message in the `Event Log`. And once the disk spaces or nodes resources are good enough, the eviction should continue. Or if the user selects `Eviction Requested` to `false`, Longhorn should stop eviction and clear the `evictionRequested` fields for nodes, disks and volumes crd objects. E.g. When there are 3 nodes in the cluster, and the volume replica count is 3, the eviction should get stuck when the `Replica Node Level Soft Anti-Affinity` is `false`. #### Integration Test Plan For `Replica Node Level Soft Anti-Affinity` is enabled, create 2 replicas on the same disk or node, and then evict this disk or node, the 2 replicas should goto another disk of node. For `Replica Node Level Soft Anti-Affinity` is disabled, create 1 replica on a disk, and evict this disk or node, the replica should goto the other disk of node. For node eviction, Longhorn will process the eviction based on the disks for the node, this is like disk eviction. After eviction success, the replica number on the evicted node should be 0. #### Error Indication During the eviction, user can click the `Replicas Number` on the `Node` page, and set which replicas are left from eviction, and click the `Replica Name` will redirect user to the `Volume` page to set if there is any error for this volume. If there is any error during the rebuild, Longhorn should display the error message from UI. The error could be `failed to schedule a replica` due to disk space or based on schedule policy, can not find a valid disk to put the replica. ### Upgrade strategy No special upgrade strategy is necessary. Once the user upgrades to the new version of `Longhorn`, these new capabilities will be accessible from the `longhorn-ui` without any special work. ================================================ FILE: enhancements/20200817-improve-node-failure-handling.md ================================================ # Improve Node Failure Handling By Automatically Force Delete Terminating Pods of StatefulSet/Deployment On Down Node ## Summary Kubernetes never force deletes pods of StatefulSet or Deployment on a down node. Since the pod on the down node wasn't removed, the volume will be stuck on the down node with it as well. The replacement pods cannot be started because the Longhorn volume is RWO (see more about access modes [here](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)), which can only be attached to one node at a time. We provide an option for users to help them automatically force delete terminating pods of StatefulSet/Deployment on the down node. After force deleting, Kubernetes will detach Longhorn volume and spin up replacement pods on a new node. ### Related Issues https://github.com/longhorn/longhorn/issues/1105 ## Motivation ### Goals The goal is to help the users to monitor node status and automatically force delete terminating pods on down nodes. Without this feature, users would have to manually force delete the pods so that new replacement pods can be started. ## Proposal Implemented a mechanism to force delete pods in the Deployment/StatefulSet on a down node. There are 4 options for `NodeDownPodDeletionPolicy`: * `DoNothing` * `DeleteStatefulSetPod` * `DeleteDeploymentPod` * `DeleteBothStatefulsetAndDeploymentPod` When the setting is enabled, Longhorn will monitor node status and force delete pods on the down node on the behalf of users. ### User Stories Before this feature is implemented, the users would have to manually monitor and force delete pods when a node down so that Longhorn volume can be detached and a new replacement pod can start. This process should be automated. After this feature is implemented, the users can have the option to allow Longhorn to monitor and force delete the pods on their behalf. ### User Experience In Detail To use this enhancement, users need to change the Longhorn setting `NodeDownPodDeletionPolicy`. The default setting is `DoNothing` which means Longhorn will not force delete any pods on a down node. As a side note, even when `NodeDownPodDeletionPolicy` is set to `do-nothing`, the [automatic VolumeAttachment removal](https://longhorn.io/docs/1.0.2/high-availability/node-failure/#volume-attachment-recovery-policy) still works so deployment pods are fine if users enable automatic `volumeattachment` removal. ### API changes No API changes. ## Design We created a new controller, `Kubernetes POD Controller`, to watch pods and nodes status and handle the force deletion. Force delete a pod when all of the below conditions are met: 1. The `NodeDownPodDeletionPolicy` and pods' owner are as in the below table: | Policy \ Kind | `StatefulSet` | `ReplicaSet` | Other | | :------------- | :----------: | :----------: | :----------: | | `DoNothing` | Don't delete | Don't delete | Don't delete | | `DeleteStatefulSetPod` | Force delete | Don't delete | Don't delete | | `DeleteDeploymentPod` | Don't delete | Force delete | Don't delete | | `DeleteBothStatefulsetAndDeploymentPod` | Force delete | Force delete | Don't delete | 1. Node containing the pod is down which is determined by the [IsNodeDownOrDeleted](https://github.com/longhorn/longhorn-manager/blob/34c40abf2dabf5f25541a36917568c37e21af3ea/datastore/longhorn.go#L1015). The function `IsNodeDownOrDeleted` checks whether the node status is `NotReady` 1. The pod is terminating (which means the pod has deletionTimestamp set) and the DeletionTimestamp has passed. 1. Pod has a PV with provisioner `driver.longhorn.io` ### Implementation Overview Same as the Design ### Test plan 1. Setup a cluster of 3 nodes 1. Install Longhorn and set `Default Replica Count = 2` (because we will turn off one node) 1. Create a StatefulSet with 2 pods using the command: ``` kubectl create -f https://raw.githubusercontent.com/longhorn/longhorn/master/examples/statefulset.yaml ``` 1. Create a volume + pv + pvc named `vol1` and create a deployment of default ubuntu named `shell` with the usage of pvc `vol1` mounted under `/mnt/vol1` 1. Find the node which contains one pod of the StatefulSet/Deployment. Power off the node #### StatefulSet ##### if `NodeDownPodDeletionPolicy ` is set to `do-nothing ` | `delete-deployment-pod` - wait till the `pod.deletionTimestamp` has passed - verify no replacement pod generated, the pod is stuck at terminating forever. ##### if `NodeDownPodDeletionPolicy ` is set to `delete-statefulset-pod ` | `delete-both-statefulset-and-deployment-pod` - wait till pod's status becomes `terminating` and the `pod.deletionTimestamp` has passed (around 7 minutes) - verify that the pod is deleted and there is a new running replacement pod. - Verify that you can access/read/write the volume on the new pod #### Deployment ##### if `NodeDownPodDeletionPolicy ` is set to `do-nothing ` | `delete-statefulset-pod` AND `Volume Attachment Recovery Policy` is `never` - wait till the `pod.deletionTimestamp` has passed - replacement pod will be stuck in `Pending` state forever - force delete the terminating pod - wait till replacement pod is running - verify that you can access `vol1` via the `shell` replacement pod under `/mnt/vol1` once it is in the running state ##### if `NodeDownPodDeletionPolicy ` is set to `do-nothing ` | `delete-statefulset-pod` AND `Volume Attachment Recovery Policy` is `wait` - wait till replacement pod is generated (default is around 6 minutes, kubernetes setting) - wait till the `pod.deletionTimestamp` has passed - verify that you can access `vol1` via the `shell` replacement pod under `/mnt/vol1` once it is in the running state - verify that the original `shell` pod is stuck in `Pending` state forever ##### if `NodeDownPodDeletionPolicy ` is set to `do-nothing ` | `delete-statefulset-pod` AND `Volume Attachment Recovery Policy` is `immediate` - wait till replacement pod is generated (default is around 6 minutes, kubernetes setting) - verify that you can access `vol1` via the `shell` replacement pod under `/mnt/vol1` once it is in the running state - verify that the original `shell` pod is stuck in `Pending` state forever ##### if `NodeDownPodDeletionPolicy ` is set to `delete-deployment-pod ` | `delete-both-statefulset-and-deployment-pod` AND `Volume Attachment Recovery Policy` is `never`| `wait`|`immediate` - wait till the `pod.deletionTimestamp` has passed - verify that the pod is deleted and there is a new running replacement pod. - verify that you can access `vol1` via the `shell` replacement pod under `/mnt/vol1` #### Other kinds - Verify that Longhorn never deletes any other pod on the down node. #### Test example One typical scenario when the enhancement has succeeded is as below. When a node (say `node-x`) goes down (assume using Kubernetes' default settings and user allows Longhorn to force delete pods): | Time | Event | | :------------- | :----------: | | 0m:00s | `node-x`goes down and stops sending heartbeats to Kubernetes Node controller | | 0m:40s | Kubernetes Node controller reports `node-x` is `NotReady`. | | 5m:40s | Kubernetes Node controller starts evicting pods from `node-x` using graceful termination (set `DeletionTimestamp` and `deletionGracePeriodSeconds = 10s/30s`) | | 5m:50s/6m:10s | Longhorn forces delete the pod of StatefulSet/Deployment which uses Longhorn volume | ### Upgrade strategy Doesn't impact upgrade. ================================================ FILE: enhancements/20200819-keep-a-local-replica-to-engine.md ================================================ # Data Locality - Option To Keep A Local Replica On The Same Node As The Engine ## Summary A Longhorn volume can be backed by replicas on some nodes in the cluster and accessed by a pod running on any node in the cluster. In the current implementation of Longhorn, the pod which uses Longhorn volume could be on a node that doesn't contain any replica of the volume. In some cases, it is desired to have a local replica on the same node as the consuming pod. In this document, we refer to the property of having a local replica as having `data locality` This enhancement gives the users option to have a local replica on the same node as the engine which means on the same node as the consuming pod. ### Related Issues https://github.com/longhorn/longhorn/issues/1045 ## Motivation ### Goals Provide users an option to try to migrate a replica to the same node as the consuming pod. ### Non-goals Another approach to achieve data locality is trying to influence Kubernetes scheduling decision so that pods get scheduled onto the nodes which contain volume's replicas. However, this is not a goal in this LEP. See https://github.com/longhorn/longhorn/issues/1045 for more discussion about this approach. ## Proposal We give user 2 options for data locality setting: `disabled` and `best-effort`. In `disabled` mode, there may be a local replica of the volume on the same node as the consuming pod or there may not be. Longhorn doesn't do anything. In `best-effort` mode, if a volume is attached to a node that has no replica, the Volume Controller will start rebuilding the replica on the node after the volume is attached. Once the rebuilding process is done, it will remove one of the other replicas to keep the replica count as specified. ### User Stories Sometimes, having `data locality` is critical. For example, when the network is bad or the node is temporarily disconnected, having local replica will keep the consuming pod running. Another case is that sometimes the application workload can do replication itself (e.g. database) and it wants to have a volume of 1 replica for each pod. Without the `data locality` feature, multiple replicas may end up on the same node which destroys the replication intention of the workload. See more in [Story 1](#story-2) In the current implementation of Longhorn, the users cannot ensure that pod will have a local replica. After the enhancement implemented, users can have options to choose among `disabled` (default setting) or `best-effort` #### Story 1 A user has three hyper-converged nodes and default settings with: `default-replica-count: 2`. He wants to ensure a pod always runs with at least one local replica would reduce the amount of network traffic needed to keep the data in sync. There does not appear to be an obvious way for him to schedule the pod using affinities. #### Story 2 A user runs a database application that can do replication itself. The database app creates multiple pods and each pod uses a Longhorn volume with `replica-count = 1`. The database application knows how to schedule pods into different nodes so that they achieve HA. The problem is that replicas of multiple volumes could land on the same node which destroys the HA capability. With the `data locality` feature we can ensure that replicas are on the same nodes with the consuming pods and therefore they are on different nodes. ### User Experience In Detail * Users create a new volume using Longhorn UI with `dataLocality` set to `best-effort`. * If users attach the volume a node which doesn't contain any replica, they will see that Longhorn migrate a local replica to the node. * Users create a storageclass with dataLocality: best-effort set * Users launch a statefulset with the storageclass. * Users will find that there is always a replica on the node where the pod resides on * Users update `dataLocality` to `disable`, detach the volume, and attach it to a node which doesn't have any replica * Users will see that Longhorn does not create a local replica on the new node. ### API changes There are 2 API changes: 1. When creating a new volume, the body of the request sent to `/v1/volumes` has a new field `dataLocality` set to either `disabled` or `best-effort`. 1. Implement a new API for users to update `dataLocality` setting for individual volume. The new API could be `/v1/volumes/?action=updateDataLocality`. This API expects the request's body to have the form `{dataLocality:}`. ## Design ### Implementation Overview There are 2 modes for `dataLocality`: 1. `disabled` is the default mode. There may be a local replica of the volume on the same node as the consuming pod or there may not be. Longhorn doesn't do anything. 1. `best-effort` mode instructs Longhorn to try to keep a local replica on the same node as the consuming pod. If Longhorn cannot keep the local replica (due to not having enough disk space, incompatible disk tags, etc...), Longhorn does not stop the volume. There are 3 settings the user can change for `data locality`: 1. Global default setting inside Longhorn UI settings. The global setting should only function as a default value, like replica count. It doesn't change any existing volume's setting 1. specify `dataLocality` mode for individual volume upon creation using UI 1. specify `dataLocality` mode as a parameter on Storage Class. Implementation steps: 1. Add a global setting `DefaultDataLocality` 1. Add the new field `DataLocality` to `VolumeSpec` 1. Modify the volume creation API so that it extracts, verifies, and sets the `dataLocality` mode for the new volume. If the volume creation request doesn't have `dataLocality` field inside its body, we use the `DefaultDataLocality` for the new volume. 1. Modify the `CreateVolume` function inside the CSI package so that it extracts, verifies, and sets the `dataLocality` mode for the new volume. This makes sure that Kubernetes can use CSI to create Longhorn volume with a specified `datLocality` mode. 1. Inside `volume controller`'s sync logic, we add a new function `ReconcileLocalReplica`. 1. When a volume enters the `volume controller`'s sync logic, function `ReconcileLocalReplica` checks the `dataLocality` mode of the volume. If the `dataLocality` is `disabled`, it will do nothing and return. 1. If the `dataLocality` is `best-effort`, `ReconcileLocalReplica` checks whether there is a local replica on the same node as the volume. 1. If there is no local replica, we create an in-memory replica struct. We don't create a replica in DS using createReplica() directly because we may need to delete the new replica if it fails to ScheduleReplicaToNode. This prevents UI from repeatedly show creating/deleting the new replica. Then we try to schedule the replica struct onto the consuming pod's node. If the scheduling fails, we don't do anything. The replica struct will be collected by Go's garbage collector. If the scheduling success, we save the replica struct to the data store. This will trigger replica rebuilding on the consuming pod's node. 1. If there already exists a local replica on the consuming pod's node, we check to see if there are more healthy replica than specified on the volume's spec. If there are more healthy replicas than specified on the volume's spec, we remove a replica on the other nodes. We prefer to delete replicas on the same disk, then replicas on the same node, then replicas on the same zone. UI modification: 1. On volume creation, add an input field for `dataLocality` 1. On volume detail page: * On the right volume info panel, add a
to display `selectedVolume.dataLocality` * On the right volume panel, in the Health row, add an icon for data locality status. Specifically, if `dataLocality=best-effort` but there is not a local replica then display a warning icon. Similar to the replica node redundancy warning [here](https://github.com/longhorn/longhorn-ui/blob/0a52c1f0bef172d8ececdf4e1e953bfe78c86f29/src/routes/volume/detail/VolumeInfo.js#L47) * In the volume's actions dropdown, add a new action to update `dataLocality` 1. In Rancher UI, add a parameter `dataLocality` when create storage class using Longhorn provisioner. ### Test plan #### Manually Test Plan 1. Create a cluster of 9 worker nodes and install Longhorn. Having more nodes helps us to be more confident because the chance of randomly scheduling a replica onto the same node as the engine is small. ##### Test volume creation with `dataLocality` is `best-effort`: 1. Create volume `testvol` with `Number of Replicas = 2` and `dataLocality` is `best-effort` 1. Attach `testvol` to a node that doesn't contain any replica. 1. Verify that Longhorn schedules a local replica to the same node as the consuming pod. After finishing rebuilding the local replica. Longhorn removes a replica on other nodes to keep the number of replicas is 2. ##### Test volume creation with `dataLocality` is `disabled`: 1. Create another volume, `testvol2` with `Number of Replicas = 2` and `dataLocality` is `disabled` 1. Attach `testvol2` to a node that doesn't contain any replica. 1. Verify that Longhorn doesn't move replica ##### Test volume creation with `dataLocality` is unspecified and `DefaultDataLocality` setting as `disabled`: 1. Leave the `DefaultDataLocality` setting as `disabled` in Longhorn UI. 1. Create another volume, `testvol3` with `Number of Replicas = 2` and `dataLocality` is empty 1. Attach `testvol3` to a node that doesn't contain any replica. 1. Verify that the `dataLocality` of `testvol3` is `disabled` and that Longhorn doesn't move replica. ##### Test volume creation with `dataLocality` is unspecified and `DefaultDataLocality` setting as `best-effort`: 1. Set the `DefaultDataLocality` setting to `best-effort` in Longhorn UI. 1. Create another volume, `testvol4` with `Number of Replicas = 2` and `dataLocality` is empty 1. Attach `testvol4` to a node that doesn't contain any replica. 1. Verify that the `dataLocality` of `testvol4` is `best-effort`. 1. Verify that Longhorn schedules a local replica to the same node as the consuming pod. After finishing rebuilding the local replica. Longhorn removes a replica on other nodes to keep the number of replicas is 2. ##### Test `updateDataLocality` from `disabled` to `best-effort`: 1. Change `dataLocality` to `best-effort` for `testvol2` 1. Verify that Longhorn schedules a local replica to the same node as the consuming pod. After finishing rebuilding the local replica. Longhorn removes a replica on other nodes to keep the number of replicas which is 2. ##### Test `updateDataLocality` from `best-effort` to `disabled` : 1. Change `dataLocality` to `disabled` for `testvol2` 1. Go to Longhorn UI, increase the `number of replicas` to 3. Wait until the new replica finishes rebuilding. 1. Delete the local replica on the same node as the consuming pod. 1. Verify that Longhorn doesn't move replica ##### Test volume creation by using storage class with `dataLocality` parameter is `disabled`: 1. Create `disabled-longhorn` storage class with from this yaml file: ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: disabled-longhorn provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "1" dataLocality: "disabled" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" ``` 1. create a deployment of 1 pod using PVC dynamically created by `disabled-longhorn` storage class. 1. The consuming pod is likely scheduled onto a different node than the replica. If this happens, verify that Longhorn doesn't move replica ##### Test volume creation by using storage class with `dataLocality` parameter is `best-effort`: 1. Create `best-effort-longhorn` storage class with from this yaml file: ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: best-effort-longhorn provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "1" dataLocality: "best-effort" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" ``` 1. create a shell deployment of 1 pod using the PVC dynamically created by `best-effort-longhorn` storage class. 1. The consuming pod is likely scheduled onto a different node than the replica. 1. If this happens, verify that Longhorn schedules a local replica to the same node as the consuming pod. After finishing rebuilding the local replica, Longhorn removes a replica on other nodes to keep the number of replicas which is 1. 1. verify that the volume CRD has `dataLocality` is `best-effort` ##### Test volume creation by using storage class with `dataLocality` parameter is unspecified`: 1. Create `unspecified-longhorn` storage class with from this yaml file: ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: unspecified-longhorn provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "1" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" ``` 1. create a shell deployment of 1 pod using PVC dynamically created by `unspecified-longhorn` storage class. 1. The consuming pod is likely scheduled onto a different node than the replica. 1. If this happens, depend on `DefaultDataLocality` setting in Longhorn UI, verify that Longhorn does/doesn't migrate a local replica to the same node as the consuming pod. ##### Tests for the volumes created in old versions: 1. The volumes created in old Longhorn versions don't have the field `dataLocality`. 1. We treat those volumes the same as having `dataLocality` set to `disabled` 1. Verify that Longhorn doesn't migrate replicas for those volumes. ### Upgrade strategy No special upgrade strategy is required. We are adding the new field, `dataLocality`, to volume CRD's spec. Then we use this field to check whether we need to migrate a replica to the same node as the consuming pod. When users upgrade Longhorn to this new version, it is possible that some volumes don't have this field. This is not a problem because we only migrate replica when `dataLocality` is `best-effort`. So, the empty `dataLocality` field is fine. ================================================ FILE: enhancements/20200821-add-revision-counter-disable-support.md ================================================ # Add Revision Counter Disable Support ## Summary This enhancement adds revision counter disable support globally and individually per each volume with a logic based on latest modified time and total block counts of volume-head image file to select the most updated replica rather than the biggest revision counter for salvage recovering. ### Related Issues https://github.com/longhorn/longhorn/issues/508 ## Motivation ### Goals 1. By default 'DisableRevisionCounter' is 'false', but Longhorn provides an optional for user to disable it. 2. Once user set 'DisableRevisionCounter' to 'true' globally or individually, this will improve Longhorn data path performance. 3. And for 'DisableRevisionCounter' is 'true', Longhorn will keep the ability to find the most suitable replica to recover the volume when the engine is faulted(all the replicas are in 'ERR' state). 4. Also during Longhorn Engine starting, with head file information it's unlikely to find out out of synced replicas. So will skip the check. ## Proposal Currently Longhorn Replica will update revision counter to the 'revision.counter' file on every write operation, this impacts the performance a lot. And the main purpose of this revision counter is to pick the most updated replica to recover the volume. Also every time when Longhorn Engine starts, it will check all the replicas' revision counter to make sure the consistency. Having an option to disable the revision counter with a new logic which selects the most updated replica based on the last modified time and the head file size of the replica, and removing the revision counter mechanism. These can improve the Longhorn performance and keep the salvage feature as the same. Longhorn Engine will not check the synchronization of replicas during the starting time. ### User Stories Some user has concern about Longhorn's performance. And with this 'DisableRevisionCounter' option, Longhorn's performance will be improved a lot. #### TODO Will update the performance data here later. ### User Experience In Detail In Longhorn UI 'Setting' 'General', by default 'DisableRevisionCounter' is 'false', user can set 'DisableRevisionCounter' to 'true'. This only impacts the UI setting. Or from StorageClass yaml file, user can set 'parameters' 'revisionCounterDisabled' to true. And all the volume created based on this storageclass will have the same 'revisionCounterDisabled' setting. User can also set 'DisableRevisionCounter' for each individual volumes created by Longhorn UI this individual setting will over write the global setting. Once the volume has 'DisableRevisionCounter' to 'true', there won't be revision counter file. And the 'Automatic salvage' is 'true', when the engine is faulted, the engine will pick the most suitable replica as 'Source of Truth' to recover the volume. ### API changes 1. Add 'DisableRevisionCounter' global setting in Longhorn UI 'Setting' 'General'. 2. Add 'DisableRevisionCounter' individual setting in Longhorn UI 'Volume' 'Create Volume' for each volume. 3. For CSI driver, need to update volume creation API. 4. Add new parameter to 'ReplicaProcessCreate' to disable revision counter. And new parameter to 'EngineProcessCreate' to indicate the salvage requested mode for recovering. 5. Update Longhorn Engine Replica proto 'message Replica' struct with two new fields 'last_modified_time' and 'head_file_size' of the head file. This is for Longhorn Engine Control to get these information from Longhorn Engine Replica. ## Design ### Implementation Overview This enhancement has two phases, the first phase is to enable the setting to disable revision counter. The second phase is the implement the new gRPC APIs with new logic for salvage. And for the API compatibility issues, always check the 'EngineImage.Statue.cliAPIVersion' before making the call. #### Disable Revision Counter 1. Add 'Volume.Spec.RevisionCounterDisabled', 'Replica.Spec.RevisionCounterDisabled' and 'Engine.Spec.RevisionCounterDisabled' to volume, replica and engine objects. 2. Once 'RevisionCounterDisabled' is 'true', volume controller will set 'Volume.Spec.RevisionCounterDisabled' to true, 'Replica.Spec.RevisionCounterDisabled' and 'Engine.Spec.RevisionCounterDisabled' will set to true. And during 'ReplicaProcessCreate' and 'EngineProcessCreate' , this will be passed to engine replica process and engine controller process to start a replica and controller without revision counter. 3. During 'ReplicaProcessCreate' and 'EngineProcessCreate', if 'Replica.Spec.RevisionCounterDisabled' or 'Engine.Spec.RevisionCounterDisabled' is true, it will pass extra parameter to engine replica to start replica without revision counter or to engine controller to start controller without revision counter support, otherwise keep it the same as current and engine replica will use the default value 'false' for this extra parameter. This is the same as the engine controller to set the 'salvageRequested' flag. 4. Add 'RevisionCounterDisabled' in 'ReplicaInfo', when engine controller start, it will get all replica information. 4. For engine controller starting cases: - If revision counter is not disabled, stay with the current logic. - If revision counter is disabled, engine will not check the synchronization of the replicas. - If unexpected case (engine controller has revision counter disabled but any of the replica doesn't, or engine controller has revision counter enabled, but any of the replica doesn't), engine controller will log this as error and mark unmatched replicas to 'ERR'. #### Add New Logic for Salvage Once the revision counter has been disabled. 1. Add 'SalvageRequested' in 'InstanceSpec' and 'SalvageExecuted' in 'InstanceStatus' to indicate salvage recovering status. If all replicas are failed, 'Volume Controller' will set 'Spec.SalvageRequested' to 'true'. 2. In 'Engine Controller' will pass 'Spec.SalvageRequested' to 'EngineProcessCreate' to trigger engine controller to start with 'salvageRequested' is 'true' for the salvage logic. 3. The salvage logic gets details of replicas to get the most suitable replica for salvage. - Based on 'volume-head-xxx.img' last modified time, to get the latest one and any one within 5 second can be put in the candidate replicas for now. - Compare the head file size for all the candidate replicas, pick the one with the most block numbers as the 'Source of Truth'. - Only mark one candidate replica to 'RW' mode, the rest of replicas would be marked as 'ERR' mode. 4. Once this is done, set 'SalvageExecuted' to 'true' to indicate the salvage is done and change 'SalvageRequested' back to false. ### Test plan #### Disable revision counter option test case Disable revision counter option should return error if only Longhorn Manager got upgraded, not Longhorn Engine. This is when user trying to disable revision counter with new Longhorn Manager but the Longhorn Engine is still the over version which doesn't have this feature support. In this case, UI will shows error message to user. #### Disable revision counter options Revision counter can be disabled globally via UI by set 'Setting' 'General' 'DisableRevisionCounter' to 'true'. It can be set locally per volume via UI by set 'Volume' 'Create Volume''DisableRevisionCounter' to true. It can be set via 'StorageClass', and every PV created by this 'StorageClass' will inherited the same setting: ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: best-effort-longhorn provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "1" disableRevisionCounter: "true" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" ``` #### Integration Test Plan 1. Disable the revision counter. 2. Create a volume with 3 replicas. 3. Attach the volume to a node, and start to write data to the volume. 4. Kill the engine process during the data writing. 5. Verify the volume still works fine. 6. Repeat the above test multiple times. ### Upgrade strategy Deploy Longhorn image with v1.0.2 and upgrade Longhorn Manager, salvage function should still work. And then update Longhorn Engine, the revision counter disabled feature should be available. ================================================ FILE: enhancements/20200821-rebuild-replica-with-existing-data.md ================================================ # Rebuild replica with existing data ## Summary Longhorn could reuse the existing data of failed replicas to speed up rebuild progress as well as save bandwidth. ### Related Issues https://github.com/longhorn/longhorn/issues/1304 ## Motivation ### Goals 1. The (data of) failed replicas can be reused during the replica rebuild. 2. The rebuild won't be blocked when the data of failed replicas are completely corrupted, or there is no existing replica. 3. With the existing data, some of the data transferring can be skipped, and replica rebuild may speed up. ## Proposal 1. Add a new setting `ReplicaReplenishmentWaitInterval` to delay the replica rebuild. - If the failed replica currently is unavailable but it may be able to be reused later(we call it potential reusable failed replica), Longhorn may need to delay the new replica replenishment so that there is a chance to reuse this kind of replica. - For eviction/data locality/new volume cases, a new replica should be recreated immediately hence this setting won't be applied. 2. In order to reuse the existing data, Longhorn can directly reuse the failed replica objects for the rebuild. 3. Add max retry count for the replica rebuild with failed replicas. Otherwise, the rebuild will get stuck of the reusing the failed replicas there if the data of failed replicas are completely corrupted. 4. Add backoff interval for the retry of the failed replica reuse. ### User Stories #### Rebuild replica for a large volume after network fluctuation/node reboot Before the enhancement, there is no chance to reuse the failed replicas on the node, and the rebuild can take a long time with heavy bandwidth usage. After the enhancement, the replica rebuild won't start until the new worker nodes with old disks are up. Then the failed replicas will be reused during the rebuild, and the rebuild can be pretty fast. ### User Experience In Detail Users don't need to do anything except for setting `ReplicaReplenishmentWaitInterval` ### API Changes No API change is required. ## Design ### Implementation Overview #### longhorn-manager: 1. Add a setting `ReplicaReplenishmentWaitInterval`. - This will block the rebuilding when there is a failed replica that is temporarily unavailable in the volume. - Add a field `volume.Status.LastDegradedAt` so that we can determine if `ReplicaReplenishmentWaitInterval` is passed. 2. Add field `Replica.Spec.RebuildRetryCount` to indicate how many times Longhorn tries to reuse this failed replica for the rebuild. 3. In Volume Controller && Replica Scheduler: 1. Check if there is a reusable failed replica and if the replica reuse is not in the backoff window. If YES, directly try to reuse the failed replica. 2. Otherwise, replenish a new replica is required for one of the following cases: 1. the volume is a new volume (volume.Status.Robustness is Empty) 2. data locality is required (hardNodeAffinity is not Empty and volume.Status.Robustness is Healthy) 3. replica eviction happens (volume.Status.Robustness is Healthy) 4. there is no potential reusable replica 5. there is a potential reusable replica but the replica replenishment wait interval is passed. 3. Reuse the failed replica by cleaning up `ReplicaSpec.HealthyAt` and `ReplicaSpec.FailedAt`. And `Replica.Spec.RebuildRetryCount` will be increased by 1. 4. Clean up the related record in `Replica.Spec.RebuildRetryCount` when the rebuilding replica becomes mode `RW`. 5. Guarantee the reused failed replica will be stopped before re-launching it. ### Test Plan #### Manually Test Plan ##### Rebuild replica for a large volume after network fluctuation/node reboot 1. Set `ReplicaReplenishmentWaitInterval`. Make sure it's longer than the node recovery interval. 2. Create and attach a large volume. Set a short `staleReplicaTimeout` for the volume, e.g., 1 minute. 3. Write a large amount of data then take a snapshot. 4. Repeat step 3 several times. 5. Reboot/Temporarily disconnect a node contains replica only. 6. According to the `ReplicaReplenishmentWaitInterval` and the node recovery interval: - Verify the failed replica is reused and there is no new replica for the rebuild after the node recovery. - Verify the replica rebuild only takes a relatively short time. ##### Replenish replicas when failed replicas cannot be reused 1. Create and attach a large volume. 2. Write data then take snapshots. 3. Hack into one replica directory and make the directory and files read-only. 4. Crash the related replica process and wait for the replica failure. 5. Wait and check if Longhorn tries to reuse the corrupted replica but always fail. Since there is backoff mechanism, this will take a long time(8 ~ 10min). 6. Check if Longhorn will create a new replica and succeeds to finish the rebuild when the max retry count is reached. 7. Verify the data content. And check if the volume still works fine. ##### Replenish replicas when failed there is a potential replica and the replenishment wait interval is passed 1. Set `ReplicaReplenishmentWaitInterval` to 60s. 2. Create and attach a large volume. 3. Write data then take snapshots. 4. Shut down a node containing replica only for 60s. 5. Wait and check if Longhorn tries to reuse the failed replica for 2~3 times but always fail. 6. Check if Longhorn will create a new replica once the replenishment wait interval is passed. 7. Verify the data content. And check if the volume still works fine. #### Reuse failed replicas for an old degraded volume after live upgrade: 1. Deploy Longhorn v1.0.2. 2. Create and attach a volume. Write data to the volume. 3. Disable scheduling for 1 node. 4. Crash the replica on the node. 5. Upgrade Longhorn to the latest. Verify the volume robustness `Degraded`. 6. Enable scheduling for the node. Verify the failed replica of the existing degraded volume will be reused. 7. Verify the data content, and the volume r/w still works fine. #### Failed replicas reusage backoff won't block replica replenishment 1. Deploy the latest Longhorn. 2. Create and attach a volume. Write data to the volume. 3. Update `Replica Replenishment Wait Interval` to 60s. 4. Crash a replica: removing the volume head file and creating a directory with the volume head file name. Then the replica reuse will continuously fail. e.g., `rm volume-head-001.img && mkdir volume-head-001.img` 5. Verify: 1. There is a backoff interval for the failed replica reuse. 2. A new replica will be created after (around) 60s despite the failed replica reuse is in backoff. 3. the data content. 4. the volume r/w still works fine. #### Integration Test Plan ##### Reuse the failed replicas when the replica data is messed up 1. Set a long wait interval for setting `replica-replenishment-wait-interval`. 2. Disable the setting soft node anti-affinity. 3. Create and attach a volume. Then write data to the volume. 4. Disable the scheduling for a node. 5. Mess up the data of a random snapshot or the volume head for a replica. Then crash the replica on the node. --> Verify Longhorn won't create a new replica on the node for the volume. 6. Update setting `replica-replenishment-wait-interval` to a small value. 7. Verify Longhorn starts to create a new replica for the volume. Notice that the new replica scheduling will fail. 8. Update setting `replica-replenishment-wait-interval` to a large value. 9. Delete the newly created replica. --> Verify Longhorn won't create a new replica on the node for the volume. 10. Enable the scheduling for the node. 11. Verify the failed replica (in step 5) will be reused. 12. Verify the volume r/w still works fine. #### Reuse the failed replicas with scheduling check 1. Set a long wait interval for setting `replica-replenishment-wait-interval`. 2. Disable the setting soft node anti-affinity. 3. Add tags for all nodes and disks. 4. Create and attach a volume with node and disk selectors. Then write data to the volume. 5. Disable the scheduling for the 2 nodes (node1 and node2). 6. Crash the replicas on the node1 and node2. --> Verify Longhorn won't create new replicas on the nodes. 7. Remove tags for node1 and the related disks. 8. Enable the scheduling for node1 and node2. 9. Verify the only failed replica on node2 is reused. 10. Add the tags back for node1 and the related disks. 11. Verify the failed replica on node1 is reused. 12. Verify the volume r/w still works fine. ### Upgrade strategy Need to update `volume.Status.LastDegradedAt` for existing degraded volumes during live upgrade. ================================================ FILE: enhancements/20200904-csi-snapshot-support.md ================================================ # CSI Snapshot Support ## Summary To allow users to create/restore/delete backups programmatically, we want to add support for the csi snapshot mechanism. This way existing tools can be used to create kubernetes `VolumeSnapshot` api resources, which we can react to in the csi plugin. ### Related Issues https://github.com/longhorn/longhorn/issues/304 https://github.com/longhorn/longhorn/issues/610 https://github.com/longhorn/longhorn/issues/1127 https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-snapshot.md ## Motivation Provide user with programmatic access for backup operations via the standardized csi interface. https://kubernetes.io/docs/concepts/storage/volume-snapshots/#provisioning-volume-snapshot ### Goals - add csi snapshot support to our csi driver ### Non-goals - VolumeBackup crd refactor - Changes to the longhorn backup code ## Proposal - support csi CreateSnapshot call - support csi DeleteSnapshot call - support snapshot as ContentSource for restoration during CreateVolume calls ### User Stories Currently, it's hard for users to interact with the backup system programmatically, after this enhancement the users will be able to use the standard kubernetes csi mechanisms for backup creation / deletion and restoration of a new volume based on a backup. ### User Experience In Detail #### Backup creation via VolumeSnapshot resource The user can request a backup of a volume by creation of a kubernetes `VolumeSnapshot` object. Example below for a volume named `test-vol` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-pvc spec: volumeSnapshotClassName: longhorn source: persistentVolumeClaimName: test-vol ``` #### Restoration via VolumeSnapshot resource The user can request the creation of a volume based on a prior created `VolumeSnapshot` object. Example below for a volume named `test-vol-restore` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-vol-restore spec: storageClassName: longhorn dataSource: name: test-vol-snapshot kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 2Gi ``` #### Restoration of an existing Longhorn backup (pre-provisioning) The user can request the creation of a volume based on a prior longhorn backup, that was not created via the csi layer. The user needs to create a `VolumeSnapshotContent` object with an associated `VolumeSnapshot` object. The `snapshotHandle` of the `VolumeSnapshotContent` needs to point to an existing longhorn backup. Example below for a volume named `test-restore-existing-backup` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotContent metadata: name: test-existing-backup spec: volumeSnapshotClassName: longhorn driver: driver.longhorn.io deletionPolicy: Delete source: # NOTE: change this to point to an existing backup on the backupstore snapshotHandle: bs://test-vol/backup-625159fb469e492e volumeSnapshotRef: name: test-snapshot-existing-backup namespace: default ``` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-existing-backup spec: volumeSnapshotClassName: longhorn source: volumeSnapshotContentName: test-existing-backup ``` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-existing-backup spec: storageClassName: longhorn dataSource: name: test-snapshot-existing-backup kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 2Gi ``` #### Backup deletion via VolumeSnapshot resource The user can request the deletion of a backup by removing the associated `VolumeSnapshot` object. If the DeletionPolicy is Delete, then the referenced longhorn backup will be deleted along with the `VolumeSnapshotContent` object. If the DeletionPolicy is Retain, then both the referenced longhorn backup and `VolumeSnapshotContent` object remain. The default DeletionPolicy is set to Delete, if the user wants to retain the longhorn backups, the user can create a snapshotClass with DeletionPolicy set to Retain. https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/#deletionpolicy Example below for a snapshot named `test-snapshot-pvc` `kubectl delete volumesnapshots test-snapshot-pvc` Deletion is triggered by deleting the VolumeSnapshot object, and the DeletionPolicy will be followed. The default deletion policy ### API changes no changes necessary ## Design ### Implementation Overview #### Implement CSI CreateSnapshot call The creation of a `VolumeSnapshot` resource triggers the creation of a longhorn snapshot afterwards, that snapshot will be backed up via the longhorn backup mechanism to a user defined backupstore (s3, nfs). While the backup isn't completed the csi snapshot will be marked not ready to use. The csi-snapshot-request name is generated by the csi snapshotter, based on the configured prefix + `VolumeSnapshot.uuid` This information is also used to generate the `VolumeSnapshotContent.name = snapcontent-uuid` The csi-snapshotter will always call the longhorn csi plugin with the same csi snapshot request name for a specific Snapshot object. This means we have to use the csi snapshotter generated name, for the longhorn snapshot or introduce a CRD (VolumeBackup) so that we have a persisted way of associating csiSnapshot - longhornSnapshot - longhornBackup If we would ignore the requested name, we would end up generating a new snapshot for each successive call, since a backup can take a long time for very big volumes, that would not be the desired behavior. To be able to lookup the backup during a following `DeleteSnapshot` or `CreateVolume` call, we encode the backupVolume and backupName as part of the snapshotID which is returned from the csi CreateSnapshot call. this will be set as the `VolumeSnapshotContent.snapshotHandle` for the kubernetes created `VolumeSnapshotContent` object. We use the format `type://backupVolume/backupName` where the default type equals `bs` for direct references to longhorn backups. This is so that in the future we can refer to a custom kubernetes resource, which we can use for backup metadata persistence. #### Implement CSI DeleteSnapshot call For backup deletion all we have todo is decode the snapshotID then trigger the backup delete calls via the longhorn api. #### Add CSI CreateVolume ContentSource support. For the volume creation based on a CSI snapshot we can decode the snapshotID to lookup the backup in the backupstore. This will provide us with the backupURL which we can add to the `fromBackup` field in the longhorn created Volume resource. As part of prior work longhorn already knows how to restore these backups, this was initially used for a StorageClass parameter, by reusing the same mechanism we don't have to maintain multiple code paths for volume restoration. ### Test plan See examples of the necessary yaml manifests in the `User Experience In Detail` section. Creation test: - create volume - write data to volume - create a `VolumeSnapshot` object - wait for `VolumeSnapshot` to be ready to use - check for backup existence on the backupstore Deletion test: - create volume - write data to volume - create a `VolumeSnapshot` object - wait for `VolumeSnapshot` to be ready to use - check for backup existence on the backupstore - delete `VolumeSnapshot` object - wait for backup removal from the backupstore Restore csi snapshot test: - create volume - write data to volume - create a `VolumeSnapshot` object - wait for `VolumeSnapshot` to be ready to use - check for backup existence on the backupstore - create PVC with content source set to the `VolumeSnapshot` object - wait for volume restoration - verify restored volume data == previously written data Restore existing longhorn backup test: - create volume - write data to volume - create a longhorn backup - check for backup existence on the backupstore - create a `VolumeSnapshotContent` object pointing to the longhorn backup - create a `VolumeSnapshot` object pointing to the `VolumeSnapshotContent` object - create PVC with content source set to the `VolumeSnapshot` object - wait for volume restoration - verify restored volume data == previously written data ### Upgrade strategy For csi snapshot support the user needs to update their kubernetes installation to at least 1.17 For environments where the user has pinned their csi images (airgap) the users need to manually provide the following images: - longhornio/csi-provisioner:v1.6.0 - longhornio/csi-snapshotter:v2.1.1 We upgraded the csi-provsioner from 1.4 to 1.6, which still supports kubernetes 1.13 as a minimum version ## Note Since we cannot assume that the users distribution has csi snapshotter support, the user needs to create a `VolumeSnapshotClass` to be able to use the csi snapshot support. Example longhorn `VolumeSnapshotClass` ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: longhorn driver: driver.longhorn.io deletionPolicy: Delete ``` The CRDs and snapshot controller installations are the responsibility of the Kubernetes distribution. See: https://kubernetes.io/docs/concepts/storage/volume-snapshots/#introduction We are discussing whether longhorn can provide these as part of the longhorn installation, but there isn't really a good way of making sure that there isn't a snapshot controller already deployed in the cluster. Make sure your cluster contains the below crds, rancher rke did not deploy them for me. https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd Make sure your cluster contains the snapshot controller, rancher rke did not deploy it for me. https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller ================================================ FILE: enhancements/20200909-prometheus-support.md ================================================ # Prometheus Support ## Summary We currently do not have a way for users to monitor and alert about events happen in Longhorn such as volume is full, backup is failed, CPU usage, memory consumption. This enhancement exports Prometheus metrics so that users can use Prometheus or other monitoring systems to monitor Longhorn. ### Related Issues https://github.com/longhorn/longhorn/issues/1180 ## Motivation ### Goals We are planing to expose 22 metrics in this release: 1. longhorn_volume_capacity_bytes 1. longhorn_volume_actual_size_bytes 1. longhorn_volume_state 1. longhorn_volume_robustness 1. longhorn_node_status 1. longhorn_node_count_total 1. longhorn_node_cpu_capacity_millicpu 1. longhorn_node_cpu_usage_millicpu 1. longhorn_node_memory_capacity_bytes 1. longhorn_node_memory_usage_bytes 1. longhorn_node_storage_capacity_bytes 1. longhorn_node_storage_usage_bytes 1. longhorn_node_storage_reservation_bytes 1. longhorn_disk_capacity_bytes 1. longhorn_disk_usage_bytes 1. longhorn_disk_reservation_bytes 1. longhorn_instance_manager_cpu_usage_millicpu 1. longhorn_instance_manager_cpu_requests_millicpu 1. longhorn_instance_manager_memory_usage_bytes 1. longhorn_instance_manager_memory_requests_bytes 1. longhorn_manager_cpu_usage_millicpu 1. longhorn_manager_memory_usage_bytes See the [User Experience In Detail](#user-experience-in-detail) section for definition of each metric. ### Non-goals We are not planing to expose 6 metrics in this release: 1. longhorn_backup_stats_number_failed_backups 1. longhorn_backup_stats_number_succeed_backups 1. longhorn_backup_stats_backup_status (status for this backup (0=InProgress,1=Done,2=Failed)) 1. longhorn_volume_io_ops 1. longhorn_volume_io_read_throughput 1. longhorn_volume_io_write_throughput ## Proposal ### User Stories Longhorn already has a great UI with many useful information. However, Longhorn doesn't have any alert/notification mechanism yet. Also, we don't have any dashboard or graphing support so that users can have overview picture of the storage system. This enhancement will address both of the above issues. #### Story 1 In many cases, a problem/issue can be quickly discovered if we have a monitoring dashboard. For example, there are many times users ask us for supporting and the problems were that the Longhorn engines were killed due to over-use CPU limit. If there is a CPU monitoring dashboard for instance managers, those problems can be quickly detected. #### Story 2 User want to be notified about abnormal event such as disk space limit approaching. We can expose metrics provide information about it and user can scrape the metrics and setup alert system. ### User Experience In Detail After this enhancement is merged, Longhorn expose metrics at end point `/metrics` in Prometheus' [text-based format](https://prometheus.io/docs/instrumenting/exposition_formats/). Users can use Prometheus or other monitoring systems to collect those metrics by scraping the end point `/metrics` in longhorn manager. Then, user can display the collected data using tools such as Grafana. User can also setup alert by using tools such as Prometheus Alertmanager. Below are the descriptions of metrics which Longhorn exposes and how users can use them: 1. longhorn_volume_capacity_bytes This metric reports the configured size in bytes for each volume which is managed by the current longhorn manager. This metric contains 2 labels (dimensions): * `node`: the node of the longhorn manager which is managing this volume * `volume`: the name of this volume Example of a sample of this metric could be: ``` longhorn_volume_capacity_bytes{node="worker-2",volume="testvol"} 6.442450944e+09 ``` Users can use this metrics to draw graph about and quickly see the big volumes in the storage system. 1. longhorn_volume_actual_size_bytes This metric reports the actual space used by each replica of the volume on the corresponding nodes This metric contains 2 labels (dimensions): * `node`: the node of the longhorn manager which is managing this volume * `volume`: the name of this volume Example of a sample of this metric could be: ``` longhorn_volume_actual_size_bytes{node="worker-2",volume="testvol"} 1.1917312e+08 ``` Users can use this metrics to the actual size occupied on disks of Longhorn volumes 1. longhorn_volume_state This metric reports the state of the volume. The states are: 1=creating, 2=attached, 3=Detached, 4=Attaching, 5=Detaching, 6=Deleting. This metric contains 2 labels (dimensions): * `node`: the node of the longhorn manager which is managing this volume * `volume`: the name of this volume Example of a sample of this metric could be: ``` longhorn_volume_state{node="worker-3",volume="testvol1"} 2 ``` 1. longhorn_volume_robustness This metric reports the robustness of the volume. Possible values are: 0=unknown, 1=healthy, 2=degraded, 3=faulted This metric contains 2 labels (dimensions): * `node`: the node of the longhorn manager which is managing this volume * `volume`: the name of this volume Example of a sample of this metric could be: ``` longhorn_volume_robustness{node="worker-3",volume="testvol1"} 1 ``` 1. longhorn_node_status This metric reports the `ready`, `schedulable`, `mountPropagation` condition for the current node. This metric contains 3 labels (dimensions): * `node` * `condition`: the name of the condition (`ready`, `schedulable`, `mountPropagation`) * `condition_reason` Example of a sample of this metric could be: ``` longhorn_node_status{condition="allowScheduling",condition_reason="",node="worker-3"} 1 longhorn_node_status{condition="mountpropagation",condition_reason="",node="worker-3"} 1 longhorn_node_status{condition="ready",condition_reason="",node="worker-3"} 1 longhorn_node_status{condition="schedulable",condition_reason="",node="worker-3"} 1 ``` Users can use this metrics to setup alert about node status. 1. longhorn_node_count_total This metric reports the total nodes in Longhorn system. Example of a sample of this metric could be: ``` longhorn_node_count_total 3 ``` Users can use this metric to detect the number of down nodes 1. longhorn_node_cpu_capacity_millicpu Report the maximum allocatable cpu on this node Example of a sample of this metric could be: ``` longhorn_node_cpu_capacity_millicpu{node="worker-3"} 2000 ``` 1. longhorn_node_cpu_usage_millicpu Report the cpu usage on this node Example of a sample of this metric could be: ``` longhorn_node_cpu_usage_millicpu{node="worker-3"} 149 ``` 1. longhorn_node_memory_capacity_bytes Report the maximum allocatable memory on this node Example of a sample of this metric could be: ``` longhorn_node_memory_capacity_bytes{node="worker-3"} 4.031217664e+09 ``` 1. longhorn_node_memory_usage_bytes Report the memory usage on this node Example of a sample of this metric could be: ``` longhorn_node_memory_usage_bytes{node="worker-3"} 1.643794432e+09 ``` 1. longhorn_node_storage_capacity_bytes Report the storage capacity of this node Example of a sample of this metric could be: ``` longhorn_node_storage_capacity_bytes{node="worker-3"} 8.3987283968e+10 ``` 1. longhorn_node_storage_usage_bytes Report the used storage of this node Example of a sample of this metric could be: ``` longhorn_node_storage_usage_bytes{node="worker-3"} 9.060212736e+09 ``` 1. longhorn_node_storage_reservation_bytes Report the reserved storage for other applications and system on this node Example of a sample of this metric could be: ``` longhorn_node_storage_reservation_bytes{node="worker-3"} 2.519618519e+10 ``` 1. longhorn_disk_capacity_bytes Report the storage capacity of this disk. Example of a sample of this metric could be: ``` longhorn_disk_capacity_bytes{disk="default-disk-8b28ee3134628183",node="worker-3"} 8.3987283968e+10 ``` 1. longhorn_disk_usage_bytes Report the used storage of this disk Example of a sample of this metric could be: ``` longhorn_disk_usage_bytes{disk="default-disk-8b28ee3134628183",node="worker-3"} 9.060212736e+09 ``` 1. longhorn_disk_reservation_bytes Report the reserved storage for other applications and system on this disk Example of a sample of this metric could be: ``` longhorn_disk_reservation_bytes{disk="default-disk-8b28ee3134628183",node="worker-3"} 2.519618519e+10 ``` 1. longhorn_instance_manager_cpu_requests_millicpu This metric reports the requested CPU resources in Kubernetes of the Longhorn instance managers on the current node. The unit of this metric is milliCPU. See more about the unit at https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units This metric contains 3 labels (dimensions): * `node` * `instance_manager` * `instance_manager_type` Example of a sample of this metric could be: ``` longhorn_instance_manager_cpu_requests_millicpu{instance_manager="instance-manager-r-61ffe369",instance_manager_type="replica",node="worker-3"} 250 ``` 1. longhorn_instance_manager_cpu_usage_millicpu This metric reports the CPU usage of the Longhorn instance managers on the current node. The unit of this metric is milliCPU. See more about the unit at https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units This metric contains 3 labels (dimensions): * `node` * `instance_manager` * `instance_manager_type` Example of a sample of this metric could be: ``` longhorn_instance_manager_cpu_usage_millicpulonghorn_instance_manager_memory_requests_bytes{instance_manager="instance-manager-r-61ffe369",instance_manager_type="replica",node="worker-3"} 0 ``` 1. longhorn_instance_manager_memory_requests_bytes This metric reports the requested memory in Kubernetes of the Longhorn instance managers on the current node. This metric contains 3 labels (dimensions): * `node` * `instance_manager` * `instance_manager_type` Example of a sample of this metric could be: ``` longhorn_instance_manager_memory_requests_bytes{instance_manager="instance-manager-e-0a67975b",instance_manager_type="engine",node="worker-3"} 0 ``` 1. longhorn_instance_manager_usage_memory_bytes This metrics reports the memory usage of the Longhorn instance managers on the current node. This metric contains 3 labels (dimensions): * `node` * `instance_manager` * `instance_manager_type` Example of a sample of this metric could be: ``` longhorn_instance_manager_memory_usage_bytes{instance_manager="instance-manager-e-0a67975b",instance_manager_type="engine",node="worker-3"} 1.374208e+07 ``` 1. longhorn_manager_cpu_usage_millicpu This metric reports the CPU usage of the Longhorn manager on the current node. The unit of this metric is milliCPU. See more about the unit at https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units This metric contains 2 labels (dimensions): * `node` * `manager` Example of a sample of this metric could be: ``` longhorn_manager_cpu_usage_millicpu{manager="longhorn-manager-x5cjj",node="phan-cluster-23-worker-3"} 15 ``` 1. longhorn_manager_memory_usage_bytes This metric reports the memory usage of the Longhorn manager on the current node. This metric contains 2 labels (dimensions): * `node` * `manager` Example of a sample of this metric could be: ``` longhorn_manager_memory_usage_bytes{manager="longhorn-manager-x5cjj",node="worker-3"} 2.7979776e+07 ``` ### API changes We add a new end point `/metrics` to exposes all longhorn Prometheus metrics. ## Design ### Implementation Overview We follow the [Prometheus best practice](https://prometheus.io/docs/instrumenting/writing_exporters/#deployment), each Longhorn manager reports information about the components it manages. Prometheus can use service discovery mechanism to find all longhorn-manager pods in longhorn-backend service. We create a new collector for each type (volumeCollector, backupCollector, nodeCollector, etc..) and have a common baseCollector. This structure is similar to the controller package: we have volumeController, nodeController, etc.. which have a common baseController. The end result is a structure like a tree: ``` a custom registry <- many custom collectors share the same base collector <- many metrics in each custom collector ``` When a scrape request is made to endpoint `/metric`, a handler gathers data in the Longhorn custom registry, which in turn gathers data in custom collectors, which in turn gathers data in all metrics. Below are how we collect data for each metric: 1. longhorn_volume_capacity_bytes We get the information about volumes' capacity by reading volume CRD from datastore. When volume move to a different node, the current longhorn manager stops reporting the vol. The volume will be reported by a new longhorn manager. 1. longhorn_actual_size_bytes We get the information about volumes' actual size by reading volume CRD from datastore. When volume move to a different node, the current longhorn manager stops reporting the vol. The volume will be reported by a new longhorn manager. 1. longhorn_volume_state We get the information about volumes' state by reading volume CRD from datastore. 1. longhorn_volume_robustness We get the information about volumes' robustness by reading volume CRD from datastore. 1. longhorn_node_status We get the information about node status by reading node CRD from datastore. Nodes don't move likes volume, so we don't have to decide which longhorn manager reports which node. 1. longhorn_node_count_total We get the information about total number node by reading from datastore 1. longhorn_node_cpu_capacity_millicpu We get the information about the maximum allocatable cpu on this node by reading Kubernetes node resource 1. longhorn_node_cpu_usage_millicpu We get the information about the cpu usage on this node from metric client 1. longhorn_node_memory_capacity_bytes We get the information about the maximum allocatable memory on this node by reading Kubernetes node resource 1. longhorn_node_memory_usage_bytes We get the information about the memory usage on this node from metric client 1. longhorn_node_storage_capacity_bytes We get the information by reading node CRD from datastore 1. longhorn_node_storage_usage_bytes We get the information by reading node CRD from datastore 1. longhorn_node_storage_reservation_bytes We get the information by reading node CRD from datastore 1. longhorn_disk_capacity_bytes We get the information by reading node CRD from datastore 1. longhorn_disk_usage_bytes We get the information by reading node CRD from datastore 1. longhorn_disk_reservation_bytes We get the information by reading node CRD from datastore 1. longhorn_instance_manager_cpu_requests_millicpu We get the information by reading instance manager Pod objects from datastore. 1. longhorn_instance_manager_cpu_usage_millicpu We get the information by using kubernetes metric client. 1. longhorn_instance_manager_memory_usage_bytes We get the information by using kubernetes metric client. 1. longhorn_instance_manager_memory_requests_bytes We get the information by reading instance manager Pod objects from datastore. 1. longhorn_manager_cpu_usage_millicpu We get the information by using kubernetes metric client. 1. longhorn_manager_memory_usage_bytes We get the information by using kubernetes metric client. ### Test plan The manual test plan is detailed at [here](https://github.com/longhorn/longhorn-tests/blob/master/docs/content/manual/release-specific/v1.1.0/prometheus_support.md) ### Upgrade strategy This enhancement doesn't require any upgrade. ================================================ FILE: enhancements/20201002-allow-recurring-backup-detached-volumes.md ================================================ # Title Allow Recurring Backup Detached Volumes ## Summary In the current Longhorn implementation, users cannot do recurring backup when volumes are detached. This enhancement gives the users an option to do recurring backup even when volumes are detached. ### Related Issues https://github.com/longhorn/longhorn/issues/1509 ## Motivation ### Goals 1. Give the users an option to allow recurring backup to happen when the volume is detached. 1. Don't overwrite the old backups with new identical backups. This happens when the volume is detached for a long time and has no new data. 1. Avoid conflict with the data locality feature if the automatically attach happens. ### Non-goals 1. The volume will not be available while the recurring backup process is happening. ## Proposal Add new global boolean setting `Allow-Recurring-Backup-While-Volume-Detached`. When the users enable `Allow-Recurring-Backup-While-Volume-Detached`, allow recurring backup for detached volumes. ### User Stories #### Story 1 Users use [OpenFaas framework](https://www.openfaas.com/). OpenFaaS (Functions as a Service) is a framework for building serverless functions with Docker and Kubernetes. Users deploy serverless functions using Longhorn as persistent storage. Users also set up recurring backup for those Longhorn volumes. OpenFaas has APIGateway that watches and manages request to serverless functions. The nature of serverless functions is that the container that those functions run on are only created when APIGateway sees client requests to those functions. When there is no demand (e.g there is no function calls because there is no users' request), OpenFaas will scale down the number of function replicas (similar to Pod concept in Kubernetes) to 0. As the result, the Longhorn volumes are attached when the functions are called and detached when no one is calling the functions. If the recurring backup can only apply to the attached volumes, there will be many miss backups. #### Story 2 From the users' perspective, when they schedule a backup, they would expect that they can see the backup to be created automatically according to a certain schedule. The expectation is, the backup store should always have the latest data of the volume. But in the case of a volume was changed then detached before the last backup was done, the expectation will not be met. Then if the volume is lost during the time that volume is detached, then we lost the changed part of data of volume since the last backup. ### User Experience In Detail Users need to turn on the global setting `Allow-Recurring-Backup-While-Volume-Detached` to enable recurring backup for detached volumes. Longhorn will automatically attaches volumes, disables volume's frontend, takes a snapshot, creates a backup, then detaches the volume. During the time the volume was attached automatically, the volume is not ready for workload. Workload will have to wait until the recurring backup finish. ### API changes There is no API change. The new global setting `Allow-Recurring-Backup-While-Volume-Detached` will use the same `/v1/settings` API. ## Design ### Implementation Overview 1. Add new global boolean setting `Allow-Recurring-Backup-While-Volume-Detached` 1. In `volume-controller`, we don't suspend volume's recurring jobs when either of the following condition match: 1. The volume is attached. 1. The volume is detached but users `Allow-Recurring-Backup-While-Volume-Detached` is set to `true`. Other than that, we suspend volume's recurring jobs. 1. Modify the cronjob to do the following: 1. Check to see if the volume is attached. 1. If the volume is attached, we follow the same process as the current implementation. 1. If the volume is detached, attach the volume to the node of the current Longhorn manager. Also, disable the volume's frontend in the attaching request. Disable the volume's frontend make sure that pod cannot use the volume during the recurring backup process. This is necessary so that we can safely detach the volume when finishing the backup. 1. Wait for the volume to be in attached state. 1. Check the size of `VolumeHead`, if it is empty, skip the backup. We don't want to overwrite the old backups with new identical backups. This happens when the volume is detached for a long time and has no new data. 1. Detach the volume when finish backup. ### Test plan #### Manual test 1. Set `Allow-Recurring-Backup-While-Volume-Detached` to `false` 1. Create a volume 1. Attach the volume, write some data to the volume 1. Detach the volume 1. Set the recurring backup for the volume on every minute 1. Wait for 2 minutes, verify that there is no new backup created 1. Set `Allow-Recurring-Backup-While-Volume-Detached` to `true` 1. Wait until the recurring job begins. 1. Verify that Longhorn automatically attaches the volume, and does backup, then detaches the volume. 1. On very subsequence minutes, verify that Longhorn automatically attaches the volume, but doesn't do backup, then detaches the volume. The reason that Longhorn does not do backup is there is no new data. 1. Delete the recurring backup 1. Create a PVC from the volume 1. Create a deployment of 1 pod using the PVC 1. Write 1GB data to the volume from the pod. 1. Scale down the deployment. The volume is detached. 1. Set the recurring backup for every 2 minutes. 1. Wait until the recurring backup starts, scale up the deployment to 1 pod. 1. Verify that pod cannot start until the recurring backup finishes. 1. On very subsequence 2 minutes, verify that Longhorn doesn't do backup. The reason that Longhorn does not do backup is there is no new data. 1. Delete the recurring job 1. Turn on data locality for the volume 1. Set the number of NumberOfReplicas to 1 1. Let say the volume is attaching to node-1. Wait until there is a healthy replica on node-1 and there is no other replica. 1. Write 200MB data to the volume. 1. Detach the volume 1. Turn off data locality for the volume 1. Attach the volume to node-2 1. Detach the volume 1. Set the recurring backup for every 1 minutes. 1. Wait until the recurring backup starts. Verify that Longhorn automatically attaches the volume to node-2, does backup, then detaches the volume. However, Longhorn doesn't trigger the replica rebuild, and there is no new replica on node-2. 1. Set `Allow-Recurring-Backup-While-Volume-Detached` to `true` ### Upgrade strategy This enhancement doesn't require an upgrade strategy. ================================================ FILE: enhancements/20201020-recover-from-volume-failure-by-delete-and-recreate-the-workload-pod.md ================================================ # Recover From Volume Failure by Delete and Recreate The Workload Pod ## Summary [The current implementation](https://github.com/longhorn/longhorn-manager/blob/ba4ca64ad03911194c8586932e9f529e19c884a4/util/util.go#L712) of the remount feature doesn't work when the workload pod uses subpath. This enhancement proposed a new way to handle volume remount which is deleting the workload Pod if it is controlled by a controller (e.g. deployment, statefulset, daemonset). By doing so, Kubernetes will restart the pod, detach, attach, and remount the volume. ### Related Issues https://github.com/longhorn/longhorn/issues/1719 ## Motivation ### Goals Make sure that when volume attached after it is detached unexpectedly or after it is auto salvaged, the workload pod can use the volume even if the containers inside the pod use subpaths. ## Solution Space ### #1 Use `findmnt` to detect the mount point of subpath The command `findmnt `can be used to find all existing mount points corresponding to the device. The example output of this command when a pod is using subpath: ``` root@worker-1:~# findmnt /dev/longhorn/pvc-1ce69c7e-90ce-41ce-a88e-8b968d9a8ff9 TARGET SOURCE FSTYPE OPTIONS /var/lib/kubelet/pods/0429305b-faf1-4668-aaed-139a2cf4989c/volumes/kubernetes.io~csi/pvc-1ce69c7e-90ce-41ce-a88e-8b968d9a8ff9/mount /dev/longhorn/pvc-1ce69c7e-90ce-41ce-a88e-8b968d9a8ff9 ext4 rw,rela /var/lib/kubelet/pods/0429305b-faf1-4668-aaed-139a2cf4989c/volume-subpaths/pvc-1ce69c7e-90ce-41ce-a88e-8b968d9a8ff9/nginx/0 /dev/longhorn/pvc-1ce69c7e-90ce-41ce-a88e-8b968d9a8ff9[/html] ext4 rw,rela ``` We can identify which mount point is subpath and which mount point mounts to the root of the volume's filesystem. Then, Longhorn can remount the mount point which mounts to the root of the volume's filesystem. ### #2 Delete the workload pod Instead of manually finding and remounting all mount points of the volume, we delete the pod that has a controller. The pod's controller will recreate it. After that, Kubernetes handles the reattachment and remount of the volume. This solves the issue that remount doesn't work when the workload pod uses subpath in PVC. ## Proposal I would like to choose approach #2, `Delete the workload pod` because it is cleaner. Manually doing remount is cumbersome and leaves duplicated mount points on the host. ### User Stories #### Story 1 Users use subpath in PVC that is bounded to Longhorn volume. When the network goes bad, the volume becomes faulty (if there is no local replica). Longhorn auto salvages the volume when the network comes back. Then Longhorn does auto-remount but [the current remount logic](https://github.com/longhorn/longhorn-manager/blob/ba4ca64ad03911194c8586932e9f529e19c884a4/util/util.go#L712) doesn't support subpath. ### User Experience In Detail When users deploy workload using controller such as deployment, statefulset, or daemonset, They are assured that volume gets reattached and remounted in case it is detached unexpectedly. What about a pod without a controller? Users have to manually delete and recreate it. ### API changes There is no API change ## Design ### Implementation Overview The idea is that `VolumeController` will set `RemountRequestedAt` when the volume needs to remount. The `KubernetesPodController` will compare `RemountRequestedAt` with the pod's `podStartTime`. If pod's `startTime` < `vol.Status.RemountRequestAt`, `KubernetesPodController` deletes the pod. We don't delete the pod immediately though. Wait until `timeNow` > `vol.Status.RemountRequestedAt` + `delayDuration` (5s). The `delayDuration` is to make sure we don't repeatedly delete the pod too fast when `vol.Status.RemountRequestedAt` is updated too quickly by `VolumeController` After `KubernetesPodController` deletes the pod, there is no need to pass the information back to `VolumeController` because there is no need to reset the field `LastRemountRequestAt` which is just an event in the past. ### Test plan 1. Deploy a storage class with parameter `numberOfReplicas: 1` and `datalocality: best-effort` 1. Deploy a statefulset with `replicas: 1` and using the above storageclass. Make sure the container in the pod template uses subpath, like this: ```yaml volumeMounts: - name: mountPath: /mnt subPath: html ``` 1. Find the node where the statefulset pods are running. Let's say `pod-1` is on `node-1`, and use `vol-1`. 1. exec into `pod-1`, create a file `test_data.txt` inside the folder `/mnt/html` 1. Kill the replica instance manager pod on `node-1`. This action simulates a network disconnection (the engine process of the PVC cannot talk to the replica process on the killed replica instance manager pod). 1. in a 2 minutes retry loop: Exec into the `pod-1`, run `ls /mnt/html`. Verify the file `test_data.txt` exists. 1. Kill the replica instance manager pod on `node-1` one more time. 1. Wait for volume to become healthy, kill the replica instance manager pod on `node-1` one more time. 1. in a 2 minutes retry loop: Exec into the `pod-1`, run `ls /mnt/html`. Verify the file `test_data.txt` exists. 1. Update `numberOfReplicas` to 3. Wait for replicas rebuilding finishes. 1. Kill the engine instance manager pod on `node-1` 1. In a 2 minutes retry loop: Exec into the `pod-1`, run `ls /mnt/html`. Verify the file `test_data.txt` exists. 1. Delete `pod-1`. 1. in a 2 minutes retry loop: Exec into the `pod-1`, run `ls /mnt/html`. Verify the file `test_data.txt` exists. ### Upgrade strategy There is no upgrade needed. ================================================ FILE: enhancements/20201106-disk-reconnection.md ================================================ # Disk Reconnection ## Summary When disks are reconnected/migrated to other Longhorn nodes, Longhorn should be able to figure out the disk reconnection and update the node ID as well as the data path for the related replicas (including failed replicas). ### Related Issues https://github.com/longhorn/longhorn/issues/1269 ## Motivation ### Goals The goal of this feature is to reuse the existing data of the failed replica when the corresponding disk is back. ### Non-Goals As for how to reuse the existing data and handle rebuild related feature, it is already implemented in #1304, which is not the intention of this enhancement. ## Proposal Identifying the disk that is previously used in Longhorn is not the the key point. The essential of this feature is that Longhorn should know where to reuse existing data of all related replicas when the disk is reconnected. In other words, the fields that indicating the replica data position should be updated when the disk is reconnected. ### User Stories #### Migrate the existing disks to new nodes Before the enhancement, there is no way to reuse the existing data when a disk is reconnected/migrated. After the enhancement, this can be done by: 1. detach the volumes using the disk 2. Reconnect the disk to the another node (both nodes keep running) 3. reattach the related volumes #### Scale down the node but reuse the disks on the node Before the enhancement, there is no chance to reuse the failed replicas on the node. After the enhancement, Longhorn will update the path and node id for all failed replicas using the disks, then Longhorn can reuse the failed replicas during rebuilding. ### User Experience In Detail #### Migrate the existing disks to new nodes 1. Detach all related volumes using the disk before the disk migration. 2. Directly move the disk to the new node (physically or in cloud vendor) and mount the disk. 3. Add the disk with the new mount point to the corresponding new Longhorn node in Longhorn Node page. 4. Attach the volumes for the workloads. #### Scale down the node but reuse the disks on the node 1. Directly shut down the node when there are replicas on the node. Then the replicas on the node will fail. 2. Move the disks on the down node to other running nodes (physically or in cloud vendor). 3. Add the disk with the new mount point to the corresponding new Longhorn node in Longhorn Node page. 4. Wait then verify the failed replicas using the disk will be reused, and the node ID & path info will be updated. ### API Changes There is no API change. ## Design ### Implementation Overview #### longhorn-manager: 1. When a disk is ready, Longhorn can list all related replicas via `replica.Spec.DiskID` then sync up node ID and path info for these replicas. - If a disk is not ready, the scheduling info will be cleaned up. Longhorn won't be confused of updating replicas if multiple disconnected disks using the same Disk UUID. - Need to add a disk related label for replicas. 2. Store DiskUUID rather than the disk name in `replica.Spec.DiskID` - Need to update `DiskID` for existing replicas during upgrade. 3. Since the disk path of a replica may get changed but the data directory name is immutable. It's better to split `replica.Spec.DataPath` to `replica.Spec.DiskPath` and `replica.Spec.DataDirectoryName`. Then it's more convenient to sync up the disk path for replicas. - Need to update the path fields for existing replicas during upgrade. ### Test Plan #### Integration Tests ##### Disk migration 1. Disable the node soft anti-affinity. 2. Create a new host disk. 3. Disable the default disk and add the extra disk with scheduling enabled for the current node. 4. Launch a Longhorn volume with 1 replica. Then verify the only replica is scheduled to the new disk. 5. Write random data to the volume then verify the data. 6. Detach the volume. 7. Unmount then remount the disk to another path. (disk migration) 8. Create another Longhorn disk based on the migrated path. 9. Verify the Longhorn disk state. - The Longhorn disk added before the migration should become "unschedulable". - The Longhorn disk created after the migration should become "schedulable". 10. Verify the replica DiskID and the path is updated. 11. Attach the volume. Then verify the state and the data. #### Manual Tests ##### Some Longhorn worker nodes in AWS Auto Scaling group is in replacement 1. Set `ReplicaReplenishmentWaitInterval`. Make sure it's longer than the time needs for node replacement. 2. Launch a Kubernetes cluster with the nodes in AWS Auto Scaling group. Then Deploy Longhorn. 3. Deploy some workloads using Longhorn volumes. 4. Wait for/Trigger the ASG instance replacement. 5. Verify new replicas won't be created before reaching `ReplicaReplenishmentWaitInterval`. 6. Verify the failed replicas are reused after the node recovery. 7. Verify if workloads still work fine with the volumes after the recovery. ##### Longhorn upgrade with node down and removal 1. Launch Longhorn v1.0.x 2. Create and attach a volume, then write data to the volume. 3. Directly remove a Kubernetes node, and shut down a node. 4. Wait for the related replicas failure. Then record `replica.Spec.DiskID` for the failed replicas. 5. Upgrade to Longhorn master 6. Verify the Longhorn node related to the removed node is gone. 7. Verify 1. `replica.Spec.DiskID` on the down node is updated and the field of the replica on the gone node is unchanged. 2. `replica.Spec.DataPath` for all replicas becomes empty. 8. Remove all unscheduled replicas. 9. Power on the down node. Wait for the failed replica on the down node being reused. 10. Wait for a new replica being replenished and available. ### Upgrade strategy Need to update disk ID and data path for existing replicas. ================================================ FILE: enhancements/20201220-rwx-volume-support.md ================================================ # RWX volume support ## Summary We want to natively support RWX volumes, the creation and usage of these rwx volumes should preferably be transparent to the user. This would make it so that there is no manual user interaction necessary, and a rwx volume just looks the same as regular Longhorn volume. This would also allow any Longhorn volume to be used as rwx, without special requirements. ### Related Issues https://github.com/longhorn/longhorn/issues/1470 https://github.com/Longhorn/Longhorn/issues/1183 ## Motivation ### Goals - support creation of RWX volumes via Longhorn - support creation of RWX volumes via RWX pvc's - support mounting NFS shares via the CSI driver - creation of a share-manager that manages and exports volumes via NFS ### Non-goals - clustered NFS (highly available) - distributed filesystem ## Proposal RWX volumes should support all operations that a regular RWO volumes supports (backup & restore, DR volume support, etc) ### User Stories #### Native RWX support Before this enhancement we create an RWX provisioner example that was using a regular Longhorn volume to export multiple NFS shares. The provisioner then created native kubernetes NFS persistent volumes, there where many limitations with this approach (multiple workload pvs on one Longhorn volume, restore & backup is iffy, etc) After this enhancement anytime a user uses an RWX pvc we will provision a Longhorn volume and expose the volume via a share-manager. The CSI driver will then mount this volume that is exported via a NFS server from the share-manager pod. ### User Experience In Detail Users can automatically provision and use an RWX volume by having their workload use an RWX pvc. Users can see the status of their RWX volumes in the Longhorn UI, same as for RWO volumes. Users can use the created RWX volume on different nodes at the same time. ### API changes - add `AccessMode` field to the `Volume.Spec` - add `ShareEndpoint, ShareState` fields to the `Volume.Status` - add a new ShareManager crd, details below ```go type ShareManagerState string const ( ShareManagerStateUnknown = ShareManagerState("unknown") ShareManagerStateStarting = ShareManagerState("starting") ShareManagerStateRunning = ShareManagerState("running") ShareManagerStateStopped = ShareManagerState("stopped") ShareManagerStateError = ShareManagerState("error") ) type ShareManagerSpec struct { Image string `json:"image"` } type ShareManagerStatus struct { OwnerID string `json:"ownerID"` State ShareManagerState `json:"state"` Endpoint string `json:"endpoint"` } ``` ### Implementation Overview #### Key Components - Volume controller is responsible for creation of share manager crd and synchronising share status and endpoint of the volume with the share manager resource. - Share manager controller will be responsible for managing share manager pods and ensuring volume attachment to the share manager pod. - Share manager pod is responsible for health checking and managing the NFS server volume export. - CSI driver is responsible for mounting the NFS export to the workload pod ![Architecture Diagram](https://longhorn.io/img/diagrams/rwx/rwx-native-architecture.png) #### From volume creation to usage When a new RWX volume with name `test-volume` is created, the volume controller will create a matching share-manager resource with name `test-volume`. The share-manager-controller will pickup this new share-manager resource and create share-manager pod with name `share-manager-test-volume` in the longhorn-system namespace, as well as a service named `test-volume` that always points to the share-manager-pod `share-manager-test-volume`. The controller will set the `State=Starting` while the share-manager-pod is Pending and not `Ready` yet. The share-manager-pod is running our share-manager image which allows for exporting a block device via ganesha (NFS server). After starting the share-manager, the application waits for the attachment of the volume `test-volume`, this is done by an availability check of the block device in the bind-mounted `/host/dev/longhorn/test-volume` folder. The actual volume attachment is handled by the share-manager-controller setting volume.spec.nodeID to the `node` of the share-manager-pod. Once the volume is attached the share-manager will mount the volume, create export config and start ganesha (NFS server). Afterwards the share-manager will do periodic health check against the attached volume and on failure of a health check the pod will terminate. The share-manager pod will become `ready` as soon as ganesha is up and running this is accomplished via a check against `/var/run/ganesha.pid`. The share-manager-controller can now update the share-manager `State=Running` and `Endpoint=nfs://service-cluster-ip/test-volume"`. The volume-controller will update the volumes `ShareState` and `ShareEndpoint` based on the values of the share-manager `State` and `Endpoint`. Once the volumes `ShareState` is `Running` the csi-driver can now successfully attach & mount the volume into the workload pods. #### Recovering from share-manager node/volume/pod failures On node failure Kubernetes will mark the share-manager-pod `share-manager-test-volume` as terminating. The share-manager-controller will mark the share-manager `State=Error` if the share-manager pod is in any state other than `Running` or `Pending`, unless the share-manager is no longer required for this volume (no workloads consuming this volume). When the share-manager `State` is `Error` the volume-controller will continuously set the volumes `RemountRequestedAt=Now` so that we will cleanup the workload pods till the share-manager is back in order. This cleanup is to force the workload pods to initiate a new connection against the NFS server. In the future we hope to reuse the NFS connection which will make this step no longer necessary. The share-manager-controller will start a new share-manager pod on a different node and set `State=Starting`. Once the pod is `Ready` the controller will set `State=Running` and the workload pods are now able to reconnect/remount again. See above for the complete flow from `Starting -> Ready -> Running -> volume share available` ### Test plan - [Manual tests for RWX feature](https://github.com/longhorn/longhorn-tests/pull/496) - [E2E tests for RWX feature](https://github.com/longhorn/longhorn-tests/pull/512) ### Upgrade strategy - User needs to add a new share-manager-image to airgapped environments - Created a [migration job](https://longhorn.io/docs/1.1.0/advanced-resources/rwx-workloads/#migration-from-previous-external-provisioner) to allow users to migrate data from previous NFS provisioner or other RWO volumes ## Information & References ### NFS ganesha details - [overview grace period and recovery](https://www.programmersought.com/article/19554210230/) - [code grace period and recovery](https://www.programmersought.com/article/53201520803/) - [code & architecture overview](https://www.programmersought.com/article/12291336147/) - [kernel server vs ganesha](https://events.static.linuxfound.org/sites/events/files/slides/Collab14_nfsGanesha.pdf) ### NFS ganesha DBUS - [DBUS-Interface](https://github.com/nfs-ganesha/nfs-ganesha/wiki/Dbusinterface#orgganeshanfsdadmin) - [DBUS-Exports](https://github.com/nfs-ganesha/nfs-ganesha/wiki/DBusExports) The dbus interface can be used to add & remove exports. As well as make the server go into the grace period. ### NFS grace period - [NFS_lock_recovery_notes](https://linux-nfs.org/wiki/index.php/NFS_lock_recovery_notes) - [lower grace period](https://www.suse.com/support/kb/doc/?id=000019374) ### Ceph ganesha - [rook crd for ganesha](https://github.com/rook/rook/blob/master/design/ceph/ceph-nfs-ganesha.md) - [ganesha driver class library](https://docs.openstack.org/manila/rocky/contributor/ganesha.html) ### NFS ganesha Active Passive Setups - [pacemaker & drbd, suse](https://documentation.suse.com/sle-ha/12-SP4/#redirectmsg) - [pacemaker & corosync, redhat](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/high_availability_add-on_administration/ch-nfsserver-haaa) ### NFS ganesha recovery backends Rados requires ceph, some more difference between kv/ng - https://bugzilla.redhat.com/show_bug.cgi?id=1557465 - https://lists.nfs-ganesha.org/archives/list/devel@lists.nfs-ganesha.org/thread/DPULRQKCGB2QQUCUMOVDOBHCPJL22QMX/ **rados_kv**: This is the original rados recovery backend that uses a key- value store. It has support for "takeover" operations: merging one recovery database into another such that another cluster node could take over addresses that were hosted on another node. Note that this recovery backend may not survive crashes that occur _during_ a grace period. If it crashes and then crashes again during the grace period, the server is likely to fail to allow any clients to recover (the db will be trashed). **rados_ng**: a more resilient rados_kv backend. This one does not support takeover operations, but it should properly survive crashes that occur during the grace period. **rados_cluster**: the new (experimental) clustered recovery backend. This one also does not support address takeover, but should be resilient enough to handle crashes that occur during the grace period. FWIW, the semantics for fs vs. fs_ng are similar. fs doesn't survive crashes that occur during the grace period either. Unless you're trying to use the dbus "grace" command to initiate address takeover in an active/active cluster, you probably want rados_ng for now. ### CSI lifecycle: ``` CreateVolume +------------+ DeleteVolume +------------->| CREATED +--------------+ | +---+----^---+ | | Controller | | Controller v +++ Publish | | Unpublish +++ |X| Volume | | Volume | | +-+ +---v----+---+ +-+ | NODE_READY | +---+----^---+ Node | | Node Stage | | Unstage Volume | | Volume +---v----+---+ | VOL_READY | +---+----^---+ Node | | Node Publish | | Unpublish Volume | | Volume +---v----+---+ | PUBLISHED | +------------+ ``` Figure 6: The lifecycle of a dynamically provisioned volume, from creation to destruction, when the Node Plugin advertises the STAGE_UNSTAGE_VOLUME capability. ### Previous Engine migration feature https://github.com/longhorn/longhorn-manager/commit/2636a5dc6d79aa12116e7e5685ccd831747639df https://github.com/longhorn/longhorn-tests/commit/3a55d6bfe633165fb6eb9553235b7d0a2e651cec ================================================ FILE: enhancements/20210111-upgrade-engine-automatically.md ================================================ # Upgrade Volume's Engine Image Automatically ## Summary Currently, users need to upgrade the engine image manually in the UI after upgrading Longhorn. We should provide an option. When it is enabled, automatically upgrade engine images for volumes when it is applicable. ### Related Issues https://github.com/longhorn/longhorn/issues/2152 ## Motivation ### Goals Reduce the amount of manual work users have to do when upgrading Longhorn by automatically upgrade engine images for volumes when they are ok to upgrade. E.g. when we can do engine live upgrade or when volume is in detaching state. ## Proposal Add a new boolean setting, `Concurrent Automatic Engine Upgrade Per Node Limit`, so users can control how Longhorn upgrade engines. The value of this setting specifies the maximum number of engines per node that are allowed to upgrade to the default engine image at the same time. If the value is 0, Longhorn will not automatically upgrade volumes' engines to default version. After user upgrading Longhorn to a new version, there will be a new default engine image and possibly a new default instance manager image. The proposal has 2 parts: 1. In which component we will do the upgrade. 2. Identify when is it ok to upgrade engine for volume to the new default engine image. For part 1, we will do the upgrade inside `engine image controller`. The controller will constantly watch and upgrade the engine images for volumes when it is ok to upgrade. For part 2, we upgrade engine image for a volume when the following conditions are met: 1. The new default engine image is ready 1. Volume is not upgrading engine image 1. The volume condition is one of the following: 1. Volume is in detached state. 1. Volume is in attached state (live upgrade). And volume is healthy. And The current volume's engine image is compatible with the new default engine image. And If volume is not a DR volume. And Volume is not expanding. ### User Stories Before this enhancement, users have to manually upgrade engine images for volume after upgrading Longhorn system to a newer version. If there are thousands of volumes in the system, this is a significant manual work. After this enhancement users either have to do nothing (in case live upgrade is possible) or they only have to scale down/up the workload (in case there is a new default IM image) ### User Experience In Detail 1. User upgrade Longhorn to a newer version. The new Longhorn version is compatible with the volume's current engine image. Longhorn automatically do live engine image upgrade for volumes 2. User upgrade Longhorn to a newer version. The new Longhorn version is not compatible with the volume's current engine image. Users only have to scale the workload down and up. This experience is similar to restart Google Chrome to use a new version. 3. Note that users need to disable this feature if they want to update the engine image to a specific version for the volumes. If `Concurrent Automatic Engine Upgrade Per Node Limit` setting is bigger than 0, Longhorn will not allow user to manually upgrade engine to a version other than the default version. ### API changes No API change is needed. ## Design ### Implementation Overview 1. Inside `engine image controller` sync function, get the value of the setting `Concurrent Automatic Engine Upgrade Per Node Limit` and assign it to concurrentAutomaticEngineUpgradePerNodeLimit variable. If concurrentAutomaticEngineUpgradePerNodeLimit <= 0, we skip upgrading. 1. Find the new default engine image. Check if the new default engine image is ready. If it is not we skip the upgrade. 1. List all volumes in Longhorn system. Select a set of volume candidate for upgrading. We select candidates that has the condition is one of the following case: 1. Volume is in detached state. 1. Volume is in attached state (live upgrade). And volume is healthy. And Volume is not upgrading engine image. And The current volume's engine image is compatible with the new default engine image. And the volume is not a DR volume. And volume is not expanding. 1. Make sure not to upgrade too many volumes on the same node at the same time. Filter the upgrading candidate set so that total number of upgrading volumes and candidates per node is not over `concurrentAutomaticEngineUpgradePerNodeLimit`. 1. For each volume candidate, set `v.Spec.EngineImage = new default engine image` to update the engine for the volume. 1. If the engine upgrade failed to complete (e.g. the v.Spec.EngineImage != v.Status.CurrentImage), we just consider it is the same as volume is in upgrading process and skip it. Volume controller will handle the reconciliation when it is possible. ### Test plan Integration test plan. Preparation: 1. set up a backup store 2. Deploy a compatible new engine image Case 1: Concurrent engine upgrade 1. Create 10 volumes each of 1Gb. 2. Attach 5 volumes vol-0 to vol-4. Write data to it 3. Upgrade all volumes to the new engine image 4. Wait until the upgrades are completed (volumes' engine image changed, replicas' mode change to RW for attached volumes, reference count of the new engine image changed, all engine and replicas' engine image changed) 5. Set concurrent-automatic-engine-upgrade-per-node-limit setting to 3 6. In a retry loop, verify that the number of volumes who is upgrading engine is always smaller or equal to 3 7. Wait until the upgrades are completed (volumes' engine image changed, replica mode change to RW for attached volumes, reference count of the new engine image changed, all engine and replicas' engine image changed, etc ...) 8. verify the volumes' data Case 2: Dr volume 1. Create a backup for vol-0. Create a DR volume from the backup 2. Try to upgrade the DR volume engine's image to the new engine image 3. Verify that the Longhorn API returns error. Upgrade fails. 4. Set concurrent-automatic-engine-upgrade-per-node-limit setting to 0 5. Try to upgrade the DR volume engine's image to the new engine image 6. Wait until the upgrade are completed (volumes' engine image changed, replicas' mode change to RW, reference count of the new engine image changed, engine and replicas' engine image changed) 7. Wait for the DR volume to finish restoring 8. Set concurrent-automatic-engine-upgrade-per-node-limit setting to 3 9. In a 2-min retry loop, verify that Longhorn doesn't automatically upgrade engine image for DR volume. Case 3: Expanding volume 1. set concurrent-automatic-engine-upgrade-per-node-limit setting to 0 2. Upgrade vol-0 to the new engine image 3. Wait until the upgrade are completed (volumes' engine image changed, replicas' mode change to RW, reference count of the new engine image changed, engine and replicas' engine image changed) 4. Detach vol-0 5. Expand the vol-0 from 1Gb to 5GB 6. Wait for the vol-0 to start expanding 7. Set concurrent-automatic-engine-upgrade-per-node-limit setting to 3 8. While vol-0 is expanding, verify that its engine is not upgraded to the default engine image 9. Wait for the expansion to finish and vol-0 is detached 10. Verify that Longhorn upgrades vol-0's engine to the default version Case 4: Degraded volume 1. set concurrent-automatic-engine-upgrade-per-node-limit setting to 0 2. Upgrade vol-1 (an healthy attached volume) to the new engine image 3. Wait until the upgrade are completed (volumes' engine image changed, replicas' mode change to RW, reference count of the new engine image changed, engine and replicas' engine image changed) 4. Increase number of replica count to 4 to make the volume degraded 5. Set concurrent-automatic-engine-upgrade-per-node-limit setting to 3 6. In a 2-min retry loop, verify that Longhorn doesn't automatically upgrade engine image for vol-1. Cleaning up: 1. Clean up volumes 2. Reset automatically-upgrade-engine-to-default-version setting in the client fixture ### Upgrade strategy No upgrade strategy is needed. ### Additional Context None ================================================ FILE: enhancements/20210125-enhanced-cpu-reservation.md ================================================ # Enhanced CPU Reservation ## Summary To make sure Longhorn system stable enough, we can apply the node specific CPU resource reservation mechanism to avoid the engine/replica engine crash due to the CPU resource exhaustion. ### Related Issues https://github.com/longhorn/longhorn/issues/2207 ## Motivation ### Goals 1. Reserve CPU resource separately for engine manager pods and replica manager pods. 2. The reserved CPU count is node specific: 1. The more allocable CPU resource a node has, the more CPUs be will reserved for the instance managers in general. 2. Allow reserving CPU resource for a specific node. This setting will override the global setting. ### Non-goals 1. Guarantee that the CPU resource is always enough, or the CPU resource reservation is always reasonable based on the volume numbers of a node. 2. Notify/Warn users CPU resource exhaustion on a node: https://github.com/longhorn/longhorn/issues/1930 ## Proposal 1. Add new fields `node.Spec.EngineManagerCPURequest` and `node.Spec.ReplicaManagerCPURequest`. This allows reserving a different amount of CPU resource for a specific node. 2. Add two settings `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU`: 1. It indicates how many percentages of CPU on a node will be reserved for one engine/replica manager pod. 2. The old settings `Guaranteed Engine CPU` will be deprecated: - This setting will be unset and readonly in the new version. - For the old Longhorn system upgrade, Longhorn will automatically set the node fields based on the old setting then clean up the old setting, so that users don't need to do anything manually as well as not affect existing instance manager pods. ### User Stories Before the enhancement, users rely on the setting `Guaranteed Engine CPU` to reserve the same amount of CPU resource for all engine managers and all replica managers on all nodes. There is no way to reserve more CPUs for the instance managers on node having more allocable CPUs. After the enhancement, users can: 1. Modify the global settings `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` to reserve how many percentage of CPUs for engine manager pods and replica manager pods, respectively. 2. Set a different CPU value for the engine/replica manager pods on some particular nodes by `Node Edit`. ### API Changes Add a new field `EngineManagerCPURequest` and `ReplicaManagerCPURequest` for node objects. ## Design ### Implementation Overview #### longhorn-manager: 1. Add 2 new settings `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU`. - These 2 setting values are integers ranging from 0 to 40. - The sum of the 2 setting values should be smaller than 40 (%) as well. 2. In Node Controller, The requested CPU resource when creating an instance manager pod: 1. Ignore the deprecated setting `Guaranteed Engine CPU`. 2. If the newly introduced node field `node.Spec.EngineManagerCPURequest`/`node.Spec.ReplicaManagerCPURequest` is not empty, the engine/replica manager pod requested CPU is determined by the field value. Notice that the field value is a milli value. 3. Else using the formula based on setting `Guaranteed Engine Manager CPU`/`Guaranteed Replica Manager CPU`: `The Reserved CPUs = The value of field "kubenode.status.allocatable.cpu" * The setting values * 0.01`. 3. In Setting Controller - The current requested CPU of an instance manager pod should keep same as `node.Spec.EngineManagerCPURequest`/`node.Spec.ReplicaManagerCPURequest`, or the value calculated by the above formula. Otherwise, the pod will be killed then Node Controller will recreate it later. 4. In upgrade - Longhorn should update `node.Spec.EngineManagerCPURequest` and `node.Spec.ReplicaManagerCPURequest` based on setting `Guaranteed Engine CPU` then clean up `Guaranteed Engine CPU`. The fields 0 means Longhorn will use the setting values directly. The setting value 0 means removing the CPU requests for instance manager pods. #### longhorn-ui: - Add 2 new arguments `Guaranteed Engine Manager CPU(Milli)` and `Guaranteed Replica Manager CPU(Milli)` in the node update page. - Hide the deprecated setting `Guaranteed Engine CPU` which is type `Deprecated`. Type `Deprecated` is a newly introduced setting type. ### Test Plan #### Integration tests - Update the existing test case `test_setting_guaranteed_engine_cpu`: - Validate the settings `Guaranteed Engine Manager CPU` controls the reserved CPUs of engine manager pods on each node. - Validate the settings `Guaranteed Replica Manager CPU` controls the reserved CPUs of replica manager pods on each node. - Validate that fields `node.Spec.EngineManagerCPURequest`/`node.Spec.ReplicaManagerCPURequest` can override the settings `Guaranteed Engine Manager CPU`/`Guaranteed Replica Manager CPU`. #### Manual tests ##### The system upgrade with the deprecated setting. 1. Deploy a cluster that each node has different CPUs. 2. Launch Longhorn v1.1.0. 3. Deploy some workloads using Longhorn volumes. 4. Upgrade to the latest Longhorn version. Validate: 1. all workloads work fine and no instance manager pod crash during the upgrade. 2. The fields `node.Spec.EngineManagerCPURequest` and `node.Spec.ReplicaManagerCPURequest` of each node are the same as the setting `Guaranteed Engine CPU` value in the old version * 1000. 3. The old setting `Guaranteed Engine CPU` is deprecated with an empty value. 5. Modify new settings `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU`. Validate all workloads work fine and no instance manager pod restart. 6. Scale down all workloads and wait for the volume detachment. 7. Set `node.Spec.EngineManagerCPURequest` and `node.Spec.ReplicaManagerCPURequest` to 0 for some node. Verify the new settings will be applied to those node and the related instance manager pods will be recreated with the CPU requests matching the new settings. 8. Scale up all workloads and verify the data as well as the volume r/w. 9. Do cleanup. ##### Test system upgrade with new instance manager 1. Prepare 3 sets of longhorn-manager and longhorn-instance-manager images. 2. Deploy Longhorn with the 1st set of images. 3. Set `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` to 15 and 24, respectively. Then wait for the instance manager recreation. 4. Create and attach a volume to a node (node1). 5. Upgrade the Longhorn system with the 2nd set of images. Verify the CPU requests in the pods of both instance managers match the settings. 6. Create and attach one more volume to node1. 7. Upgrade the Longhorn system with the 3rd set of images. 8. Verify the pods of the 3rd instance manager cannot be launched on node1 since there is no available CPU for the allocation. 9. Detach the volume in the 1st instance manager pod. Verify the related instance manager pods will be cleaned up and the new instance manager pod can be launched on node1. ### Upgrade strategy N/A ================================================ FILE: enhancements/20210216-volume-live-migration.md ================================================ # Volume live migration ## Summary To enable Harvester to utilize Kubevirts live migration support, we need to allow for volume live migration, so that a Kubevirt triggered migration will lead to a volume migration from the old node to the new node. ### Related Issues - https://github.com/longhorn/longhorn/issues/2127 - https://github.com/rancher/harvester/issues/384 - https://github.com/longhorn/longhorn/issues/87 ## Motivation ### Goals - Support Harvester VM live migration ### Non-goals - Using multiple engines for faster volume failover for other scenarios than live migration ## Proposal We want to add volume migration support so that we can use the VM live migration support of Kubevirt via Harvester. By limiting this feature to that specific use case we can use the csi drivers attach / detach flow to implement migration interactions. To do this, we need to be able to start a second engine for a volume on a different node that uses matching replicas of the first engine. We only support this for a volume while it is used with `volumeMode=BLOCK`, since we don't support concurrent writes and having kubernetes mount a filesystem even in read only mode can potentially lead to a modification of the filesystem (metadata, access time, journal replay, etc). ### User Stories Previously the only way to support live migration in Harvester was using a Longhorn RWX volume that meant dealing with NFS and it's problems, instead we want to add support for live migration for a traditional Longhorn volume this was previously implemented for the old RancherVM. After this enhancement Longhorn will support a special `migratable` flag that allows for a Longhorn volume to be live migrated from one node to another. The assumption here is that the initial consumer will never write again to the block device once the new consumer takes over. ### User Experience In Detail #### Creating a migratable storage class To test one needs to create a storage class with `migratable: "true"` set as a parameter. Afterwards an RWX PVC is necessary since migratable volumes need to be able to be attached to multiple nodes. ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-migratable provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" migratable: "true" ``` #### Testing Kubevirt VM live migration We use CirOS as our test image for the live migration. The login account is `cirros` and the password is `gocubsgo`. To test with Harvester one can use the below example yamls as a quick start. Deploy the below yaml so that Harvester will download the CirrOS image into the local Minio store. NOTE: The CirrOS servers don't support the range request which Kubevirt importer uses, which is why we let harvester download the image first. ```yaml apiVersion: harvester.cattle.io/v1alpha1 kind: VirtualMachineImage metadata: name: image-jxpnq namespace: default spec: displayName: cirros-0.4.0-x86_64-disk.img url: https://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img ``` Afterwards deploy the `cirros-rwx-blk.yaml` to create a live migratable virtual machine. ```yaml apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachine metadata: labels: harvester.cattle.io/creator: harvester name: cirros-rwx-blk spec: dataVolumeTemplates: - apiVersion: cdi.kubevirt.io/v1alpha1 kind: DataVolume metadata: annotations: cdi.kubevirt.io/storage.import.requiresScratch: "true" name: cirros-rwx-blk spec: pvc: accessModes: - ReadWriteMany resources: requests: storage: 8Gi storageClassName: longhorn-migratable volumeMode: Block source: http: certConfigMap: importer-ca-none url: http://minio.harvester-system:9000/vm-images/image-jxpnq # locally downloaded cirros image running: true template: metadata: annotations: harvester.cattle.io/diskNames: '["cirros-rwx-blk"]' harvester.cattle.io/imageId: default/image-jxpnq labels: harvester.cattle.io/creator: harvester harvester.cattle.io/vmName: cirros-rwx-blk spec: domain: cpu: cores: 1 sockets: 1 threads: 1 devices: disks: - disk: bus: virtio name: disk-0 inputs: [] interfaces: - masquerade: {} model: virtio name: default machine: type: q35 resources: requests: memory: 128M hostname: cirros-rwx-blk networks: - name: default pod: {} volumes: - dataVolume: name: cirros-rwx-blk name: disk-0 ``` Once the `cirros-rwx` virtual machine is up and running deploy the `cirros-rwx-migration.yaml` to initiate a virtual machine live migration. ```yaml apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstanceMigration metadata: name: cirros-rwx-blk spec: vmiName: cirros-rwx-blk ``` ### API changes - volume detach call now expects a `detachInput { hostId: "" }` if `hostId==""` it will be treated as detach from all nodes same behavior as before. - csi driver now calls volume attach/detach for all volume types: RWO, RWX (NFS), RWX (Migratable). - the api volume-manager now determines, whether attach/detach is necessary and valid instead of the csi driver. #### Attach changes 1. If a volume is already attached (to the requested node) we will return the current volume. 2. If a volume is mode RWO, it will be attached to the requested node, unless it's attached already to a different node. 3. If a volume is mode RWX (NFS), it will only be attached when requested in maintenance mode. Since in other cases the volume is controlled by the share-manager. 4. If a volume is mode RWX (Migratable), will initially be attached to the requested node unless already attached, at which point a migration will be initiated to the new node. #### Detach changes 1. If a volume is already detached (from all, from the requested node) we will return the current volume. 2. If a volume is mode RWO, It will be detached from the requested node. 3. If a volume is mode RWX (NFS), it will only be detached if it's currently attached in maintenance mode. Since in other cases the volume is controlled by the share-manager. 4. If a volume is mode RWX (Migratable) It will be detached from the requested node. if a migration is in progress then depending on the requested node to detach different migration actions will happen. A migration confirmation will be triggered if the detach request is for the first node. A migration rollback will be triggered if the detach request is for the second node. ## Design ### Implementation Overview #### Volume migration flow The live migration intention is triggered and evaluated via the attach/detach calls. The expectation is that Kubernetes will bring up a new pod that requests attachment of the already attached volume. This will initiate the migration start, after this there are two things that can happen. Either Kubernetes will terminate the new pod which is equivalent to a migration rollback, or the old pod will be terminated which is equivalent to a migration complete operation. 1. Users launch a new VM with a new migratable Longhorn volume -> A migratable volume is created then attached to node1. Similar to regular attachment, Longhorn will set `v.spec.nodeID` to `node1` here. 2. Users launch the 2nd VM (pod) with the same Longhorn volume -> 1. Kubernetes requests that the volume (already attached) be attached to node2. Then Longhorn receives the attach call and set `v.spec.migrationNodeID` to `node2` with `v.spec.nodeID = node1`. 2. Longhorn volume-controller brings up the new engine on node2, with inactive matching replicas (same as live engine upgrade) 3. Longhorn CSI driver polls for the existence of the second engine on node2 before acknowledging attachment success. 3. Once the migration is started (running engines on both nodes), the following detach decides whether migration is completed successfully, or a migration rollback is desired: 1. If succeeded: Kubevirt will remove the original pod on `node1`, this will lead to requesting detachment from node1, which will lead to longhorn setting `v.spec.nodeID` to `node2` and unsetting `v.spec.migrationNodeID` 2. If failed: Kubevirt will terminate the new pod on `node2`, this will lead to requesting detachment from node2, which will lead to longhorn keeping `v.spec.nodeID` to `node1` and unsetting `v.spec.migrationNodeID` 4. Longhorn volume controller then cleans up the second engine and switches the active replicas to be the current engine ones. In summary: ``` n1 | vm1 has the volume attached (v.spec.nodeID = n1) n2 | vm2 requests attachment [migrationStart] -> (v.spec.migrationNodeID = n2) volume-controller brings up new engine on n2, with inactive matching replicas (same as live engine upgrade) csi driver polls for existence of second engine on n2 before acknowledging attach The following detach decides whether a migration is completed successfully, or a migration rollback is desired. n1 | vm1 requests detach of n1 [migrationComplete] -> (v.spec.nodeID = n2, v.spec.migrationNodeID = "") n2 | vm2 requests detach of n2 [migrationRollback] -> (v.spec.NodeID = n1, v.spec.migrationNodeID = "") The volume controller then cleans up the second engine and switches the active replicas to be the current engine ones. ``` ### Test plan #### E2E tests - E2E test for migration successful - E2E test for migration rollback ### Upgrade strategy Requires using a storage class with `migratable: "true"` parameter for the harvester volumes as well as an RWX PVC to allow live migration in Kubernetes/Kubevirt. ================================================ FILE: enhancements/20210510-automatic-rebalance-replica.md ================================================ # Automatic Replica Rebalance Have a setting for global and volume-specific settings to enable/disable automatic distribution of the off-balanced replicas when a node is newly available to the cluster. ## Summary When nodes are offline with setting `Replica Zone Level Soft Anti-Affinity` enabled, or `Replica Node Level Soft Anti-Affinity` enabled, the replicas could be duplicated and retained on the same zone/node if the remain cluster zone/node number is less than the replica number. Currently, the user needs to be aware when nodes are offline and a new node is added back to the cluster to manually delete the replicas to rebalance onto the newly available node. This enhancement proposes is to add a new Longhorn global setting `Replica Auto Balance` to enable detection and deletion of the unbalanced replica to achieve automatic rebalancing. And this enhancement also proposes to add a new setting in volume spec `replicaAutoBalance` to enable/disable automatic replica rebalancing for individual volume. ### Related Issues - https://github.com/longhorn/longhorn/issues/587 - https://github.com/longhorn/longhorn/issues/570 ## Motivation ### Goals - Support global replica automatic balancing. - Support individual volume replica automatic balancing. ### Non-goals [optional] [Clean up old Data after node failure](https://github.com/longhorn/longhorn/issues/685#issuecomment-817121946). ## Proposal Add a new global setting `Replica Auto Balance` to enable replica automatic balancing when a node is newly available to the cluster. Add a new setting in volume spec `replicaAutoBalance` to enable/disable automatic replica rebalancing for individual volume. ### User Stories Before this enhancement, the user needs to check and manually delete replicas to have deleted replicas reschedule to newly available nodes. If the user does not take action, this could lead to no redundancy volumes. After this enhancement, the user does not need to worry about manually balancing replicas when there are newly available nodes. #### Story 1 - node temporary offline As a system administrator, When a cluster node is offline and comes back online after some time, I want Longhorn to automatically detect and reschedule replicas evenly to all nodes. So I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. #### Story 2 - add new node As a system administrator, When a cluster node is offline and a new node is added, I want Longhorn to automatically detect and reschedule replicas evenly to all nodes So I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. #### Story 3 - zone temporary offline As a system administrator, When a cluster zone is offline and comes back online after some time, I want Longhorn to automatically detect and reschedule replicas evenly across zones. So I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. #### Story 4 - replica automatic rebalance for individual volume As a system administrator, When a cluster node is offline and a new node is added, I want Longhorn to automatically detect and reschedule replicas evenly to some volumes only. So I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. #### Story 5 - replica automatic rebalance for minimal redundancy As a system administrator, When multiple cluster node are offline and a new node is added, I want have an option to have Longhorn to automatically detect and reschedule only 1 replica to the new node to achieve minimal redundancy. So I do not have to worry about over resource consumption during rebalancing and I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. #### Story 6 - replica automatic rebalance for even redundancy As a system administrator, When multiple cluster node are offline and a new node is added, I want have an option to have Longhorn to automatically detect and reschedule replica to the new node to achieve even replica number redundancy. So I do not end up with uneven number of replicas across cluster nodes and I do not have to worry and standby 24/7 to check and manually rebalance replicas when there are newly available nodes. ### User Experience In Detail #### Story 1 - Replica Node Level Unbalanced With an example of 3 nodes in cluster and default of 3 replicas volume: 1. When the user enables `replica-soft-anti-affinity` and deploys PVC and Pod on node-1. The replica is distributed evenly to all 3 nodes at this point. 2. In case of node-2 is offline, the user will see replica was on node-2 gets re-scheduled to node-1 or node-3. And also a warning icon and note will appear on UI `Limited node redundancy: at least one healthy replica is running at the same node as another`. **Before enhancement** 3. Bring node-2 online, no change will be done automatically by Longhorn. The user will still see the same warning on UI. 4. To rebalance replicas, the user needs to find and delete the duplicated replica to trigger the schedule onto node-2. **After enhancement** 3. Longhorn automatically detects and deletes the duplicated replica so the replica will be scheduled onto node-2. User will see the duplicated replica get rescheduled back to node-2 and UI will see volume `Healthy` with the note `Limited node redundancy: at least one healthy replica is running at the same node as another` removed. #### Story 2 - Replica Zone Level Unbalanced With an example of cluster set for 2 zones and default of 2 replicas volume: 1. When the user enables `replica-soft-anti-affinity` and `replica-zone-soft-anti-affinity` and deploys PVC and deployment. The replica is distributed evenly to all 2 zones at this point. 2. In case of zone-2 is offline, the user will see replica was on zone-2 gets re-scheduled to zone-1. **Before enhancement** 3. Bring zone-2 online, no change will be done automatically by Longhorn. The user will still see replicas all running in zone-1. 4. To rebalance replicas, the user needs to find and delete the duplicated replica to trigger the schedule to zone-2. **After enhancement** 3. Longhorn automatically detects and deletes the duplicated replica so the replica will be scheduled to zone-2. The user will see the duplicated replica get rescheduled back to zone-2. ### API changes - The new global setting `Replica Auto Balance` will use the same /v1/settings API. - When creating a new volume, the body of the request sent to /v1/volumes has a new field `replicaAutoBalance` set to `ignored`, `disabled`, `least-effort`, or `best-effort`. - Implement a new API for users to update `replicaAutoBalance` setting for individual volume. The new API could be /v1/volumes/?action=updateReplicaAutoBalance. This API expects the request's body to have the form {replicaAutoBalance:}. ## Design ### Implementation Overview #### longhorn-manager - Add new global setting `Replica Auto Balance`. - The setting is `string`. - Available values are: `disabled`, `least-effort`, `best-effort`. - `disabled`: no replica auto balance will be done. - `least-effort`: replica will be balanced to achieve minimal redundancy. For example, after adding node-2, a volume with 4 off-balanced replicas will only rebalance 1 replicas. ``` node-1 +-- replica-a +-- replica-b +-- replica-c node-2 +-- replica-d ``` - `best-effort`: replica will be balanced to achieve similar number of replicas redundancy. For example, after adding node-2, a volume with 4 off-balanced replicas will rebalance 2 replicas. ``` node-1 +-- replica-a +-- replica-b node-2 +-- replica-c +-- replica-d ``` - The default value is `disabled`. - Add new volume spec `replicaAutoBalance`. - Available values are: `ignored`, `disabled`, `least-effort`, `best-effort`. - `ignored`: This will adopt to the value from global setting. - `disabled`: Same as global setting value `disabled`. - `least-effort`: Same as global setting value `least-effort`. - `best-effort`: Same as global setting value `best-effort`. - The default value is `ignored`. - In Volume Controller `syncVolume` -> `ReconcileEngineReplicaState` -> `replenishReplicas`, calculate and add number of replicas to be rebalanced to `replenishCount`. > The logic ignores all `soft-anti-affinity` settings. This will always try to achieve zone balance then node balance. And creating for replicas will leave for ReplicaScheduler to determine for the candidates. 1. Skip volume replica rebalance when volume spec `replicaAutoBalance` is `disabled`. 2. Skip if volume `Robustness` is not `healthy`. 3. For `least-effort`, try to get the replica rebalance count. 1. For `zone` duplicates, get the replenish number. 1. List all the occupied node zones with volume replicas running. - The zone is balanced when this is equal to volume spec `NumberOfReplicas`. 2. List all available and schedulable nodes in non-occupied zones. - The zone is balanced when no available nodes are found. 3. Get the number of replicas off-balanced: - number of replicas in volume spec - number of occupied node zones. 4. Return the number to replenish. - number of non-occupied zones if less than off-balanced, or - number off-balanced. 2. For `node` duplicates, try to balance `zone` first. Get the replica replenish number. 1. List all occupied node IDs with volume replicas running. - The node is balanced when this is equal to volume spec `NumberOfReplicas`. 2. List all available and schedulable nodes. - The nodes is balanced when number of occupied nodes equal to the number of nodes. This is to determine if balanced when the number of nodes is less then the volume spec `NumberOfReplicas`. 3. Get the number of replicas off-balanced: - number of replicas in volume spec - number of occupied node IDs. 4. Return the number to replenish. - number of non-occupied nodes if less than off-balanced, or - number off-balanced. 4. For `best-effort`, try `least-effort` first to achieve minimal redundancy, then, 1. Try to get zone duplicates mapped by zoneID, continue to find duplicates on nodes if no duplicated found here. 2. Try to get node duplicates mapped by nodeID. 3. Return number to replenish when maximum replica names in duplicates mapping is 2 greater than the minimum replica names in duplicates mapping. 5. Add the number to rebalance to `replenishCount` in `replenishReplicas`. - Cleanup extra replicas for auto-balance in `cleanupExtraHealthyReplicas`. 1. Get replica names. - For `best-effort`, use the replica names from duplicates in the most duplicated zones/nodes. - For `least-effort`, use the replicas names from `getPreferredReplicaCandidatesForDeletion`. 3. Delete one replicas from the replica names. ### Test plan #### Integration tests - test_replica_auto_balance_node_least_effort Scenario: replica auto-balance nodes with `least_effort`. Given set `replica-soft-anti-affinity` to `true`. And set `replica-auto-balance` to `least_effort`. And create a volume with 6 replicas. And attach the volume to node-1. And wait for the volume to be healthy. And write some data to the volume. And disable scheduling for node-2. disable scheduling for node-3. And And count replicas running on each nodes. And 6 replicas running on node-1. 0 replicas running on node-2. 0 replicas running on node-3. When enable scheduling for node-2. Then count replicas running on each nodes. And node-1 replica count != node-2 replica count. node-2 replica count != 0. node-3 replica count == 0. And sleep 10 seconds, to ensure no addition scheduling is happening. And count replicas running on each nodes. And number of replicas running should be the same. When enable scheduling for node-3. And count replicas running on each nodes. And node-1 replica count != node-3 replica count. node-2 replica count != 0. node-3 replica count != 0. And sleep 10 seconds, to ensure no addition scheduling is happening. And count replicas running on each nodes. And number of replicas running should be the same. When check the volume data. And volume data should be the same as written. #### Integration tests - test_replica_auto_balance_node_best_effort Scenario: replica auto-balance nodes with `best_effort`. Given set `replica-soft-anti-affinity` to `true`. And set `replica-auto-balance` to `best_effort`. And create a volume with 6 replicas. And attach the volume to node-1. And wait for the volume to be healthy. And write some data to the volume. And disable scheduling for node-2. disable scheduling for node-3. And And count replicas running on each node. And 6 replicas running on node-1. 0 replicas running on node-2. 0 replicas running on node-3. When enable scheduling for node-2. And count replicas running on each node. Then 3 replicas running on node-1. 3 replicas running on node-2. 0 replicas running on node-3. And sleep 10 seconds, to ensure no addition scheduling is happening. And count replicas running on each node. And 3 replicas running on node-1. 3 replicas running on node-2. 0 replicas running on node-3. When enable scheduling for node-3. And count replicas running on each node. Then 2 replicas running on node-1. 2 replicas running on node-2. 2 replicas running on node-3. And sleep 10 seconds, to ensure no addition scheduling is happening. And count replicas running on each node. And 2 replicas running on node-1. 2 replicas running on node-2. 2 replicas running on node-3. When check the volume data. And volume data should be the same as written. #### Integration tests - test_replica_auto_balance_disabled_volume_spec_enabled Scenario: replica should auto-balance individual volume when global setting `replica-auto-balance` is `disabled` and volume spec `replicaAutoBalance` is `least_effort`. Given set `replica-soft-anti-affinity` to `true`. And set `replica-auto-balance` to `least_effort`. And disable scheduling for node-2. disable scheduling for node-3. And create volume-1 with 3 replicas. create volume-2 with 3 replicas. And set volume-2 spec `replicaAutoBalance` to `least-effort`. And attach volume-1 to node-1. attach volume-2 to node-1. And wait for volume-1 to be healthy. wait for volume-2 to be healthy. And volume-1 replicas should be running on node-1. volume-2 replicas should be running on node-1. And write some data to volume-1. write some data to volume-2. When enable scheduling for node-2. enable scheduling for node-3. And count replicas running on each nodes for volume-1. count replicas running on each nodes for volume-2. Then volume-1 replicas should be running on node-1. And volume-1 should have 3 replicas running. And volume-2 replicas should be running on node-1, node-2, node-3. And volume-2 should have 3 replicas running. And volume-1 data should be the same as written. And volume-2 data should be the same as written. #### Integration tests - test_replica_auto_balance_zone_least_effort Scenario: replica auto-balance zones with least-effort. Given set `replica-soft-anti-affinity` to `true`. And set `replica-zone-soft-anti-affinity` to `true`. And set volume spec `replicaAutoBalance` to `least-effort`. And set node-1 to zone-1. set node-2 to zone-2. set node-3 to zone-3. And disable scheduling for node-2. disable scheduling for node-3. And create a volume with 6 replicas. And attach the volume to node-1. And 6 replicas running in zone-1. 0 replicas running in zone-2. 0 replicas running in zone-3. When enable scheduling for node-2. And count replicas running on each node. And zone-1 replica count != zone-2 replica count. zone-2 replica count != 0. zone-3 replica count == 0. When enable scheduling for node-3. And count replicas running on each node. And zone-1 replica count != zone-3 replica count. zone-2 replica count != 0. zone-3 replica count != 0. #### Integration tests - test_replica_auto_balance_zone_best_effort Scenario: replica auto-balance zones with best-effort. Given set `replica-soft-anti-affinity` to `true`. And set `replica-zone-soft-anti-affinity` to `true`. And set volume spec `replicaAutoBalance` to `best-effort`. And set node-1 to zone-1. set node-2 to zone-2. set node-3 to zone-3. And disable scheduling for node-2. disable scheduling for node-3. And create a volume with 6 replicas. And attach the volume to node-1. And 6 replicas running in zone-1. 0 replicas running in zone-2. 0 replicas running in zone-3. When enable scheduling for node-2. And count replicas running on each node. And 3 replicas running in zone-1. 3 replicas running in zone-2. 0 replicas running in zone-3. When enable scheduling for node-3. And count replicas running on each node. And 2 replicas running in zone-1. 2 replicas running in zone-2. 2 replicas running in zone-3. #### Integration tests - test_replica_auto_balance_node_duplicates_in_multiple_zones Scenario: replica auto-balance to nodes with duplicated replicas in the zone. Given set `replica-soft-anti-affinity` to `true`. And set `replica-zone-soft-anti-affinity` to `true`. And set volume spec `replicaAutoBalance` to `least-effort`. And set node-1 to zone-1. set node-2 to zone-2. And disable scheduling for node-3. And create a volume with 3 replicas. And attach the volume to node-1. And zone-1 and zone-2 should contain 3 replica in total. When set node-3 to the zone with duplicated replicas. And enable scheduling for node-3. Then count replicas running on each node. And 1 replica running on node-1 1 replica running on node-2 1 replica running on node-3. And count replicas running in each zone. And total of 3 replicas running in zone-1 and zone-2. ### Upgrade strategy There is no upgrade needed. ## Note [optional] `None` ================================================ FILE: enhancements/20210525-async-pull-backups.md ================================================ # Async Pull/Push Backups From/To The Remote Backup Target ## Summary Currently, Longhorn uses a blocking way for communication with the remote backup target, so there will be some potential voluntary or involuntary factors (ex: network latency) impacting the functions relying on remote backup target like listing backups or even causing further [cascading problems](###related-issues) after the backup target operation. This enhancement is to propose an asynchronous way to pull backup volumes and backups from the remote backup target (S3/NFS) then persistently saved via cluster custom resources. This can resolve the problems above mentioned by asynchronously querying the list of backup volumes and backups from the remote backup target for final consistent available results. It's also scalable for the costly resources created by the original blocking query operations. ### Related Issues - https://github.com/longhorn/longhorn/issues/1761 - https://github.com/longhorn/longhorn/issues/1955 - https://github.com/longhorn/longhorn/issues/2536 - https://github.com/longhorn/longhorn/issues/2543 ## Motivation ### Goals Decrease the query latency when listing backup volumes _or_ backups in the circumstances like lots of backup volumes, lots of backups, or the network latency between the Longhorn cluster and the remote backup target. ### Non-goals - Automatically adjust the backup target poll interval. - Supports multiple backup targets. - Supports API pagination for listing backup volumes and backups. ## Proposal 1. Change the [longhorn/backupstore](https://github.com/longhorn/backupstore) _list_ command behavior, and add _inspect-volume_ and _head_ command. - The `backup list` command includes listing all backup volumes and the backups and read these config. We'll change the `backup list` behavior to perform list only, but not read the config. - Add a new `backup inspect-volume` command to support read backup volume config. - Add a new `backup head` command to support get config metadata. 2. Create a BackupTarget CRD _backuptargets.longhorn.io_ to save the backup target URL, credential secret, and poll interval. 3. Create a BackupVolume CRD _backupvolumes.longhorn.io_ to save to backup volume config. 4. Create a Backup CRD _backups.longhorn.io_ to save the backup config. 5. At the existed `setting_controller`, which is responsible creating/updating the default BackupTarget CR with settings `backup-target`, `backup-target-credential-secret`, and `backupstore-poll-interval`. 6. Create a new controller `backup_target_controller`, which is responsible for creating/deleting the BackupVolume CR. 7. Create a new controller `backup_volume_controller`, which is responsible: 1. deleting Backup CR and deleting backup volume from remote backup target if delete BackupVolume CR event comes in. 2. updating BackupVolume CR status. 3. creating/deleting Backup CR. 8. Create a new controller `backup_controller`, which is responsible: 1. deleting backup from remote backup target if delete Backup CR event comes in. 2. calling longhorn engine/replica to perform snapshot backup to the remote backup target. 3. updating the Backup CR status, and deleting backup from the remote backup target. 9. The HTTP endpoints CRUD methods related to backup volume and backup will interact with BackupVolume CR and Backup CR instead interact with the remote backup target. ### User Stories Before this enhancement, when the user's environment under the circumstances that the remote backup target has lots of backup volumes _or_ backups, and the latency between the longhorn manager to the remote backup target is high. Then when the user clicks the `Backup` on the GUI, the user might hit list backup volumes _or_ list backups timeout issue (the default timeout is 1 minute). We choose to not create a new setting for the user to increase the list timeout value is because the browser has its timeout value also. Let's say the list backups needs 5 minutes to finish. Even we allow the user to increase the longhorn manager list timeout value, we can't change the browser default timeout value. Furthermore, some browser doesn't allow the user to change default timeout value like Google Chrome. After this enhancement, when the user's environment under the circumstances that the remote backup target has lots of backup volumes _or_ backups, and the latency between the longhorn manager to the remote backup target is high. Then when the user clicks the `Backup` on the GUI, the user can eventually list backup volumes _or_ list backups without a timeout issue. #### Story 1 The user environment is under the circumstances that the remote backup target has lots of backup volumes and the latency between the longhorn manager to the remote backup target is high. Then, the user can list all backup volumes on the GUI. #### Story 2 The user environment is under the circumstances that the remote backup target has lots of backups and the latency between the longhorn manager to the remote backup target is high. Then, the user can list all backups on the GUI. #### Story 3 The user creates a backup on the Longhorn GUI. Now the backup will create a Backup CR, then the backup_controller reconciles it to call Longhorn engine/replica to perform a backup to the remote backup target. ### User Experience In Detail None. ### API changes 1. For [longhorn/backupstore](https://github.com/longhorn/backupstore) The current [longhorn/backupstore](https://github.com/longhorn/backupstore) list and inspect command behavior are: - `backup ls --volume-only`: List all backup volumes and read it's config (`volume.cfg`). For example: ```json $ backup ls s3://backupbucket@minio/ --volume-only { "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": { "Name": "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "Size": "2147483648", "Labels": {}, "Created": "2021-05-12T00:52:01Z", "LastBackupName": "backup-c5f548b7e86b4b56", "LastBackupAt": "2021-05-17T05:31:01Z", "DataStored": "121634816", "Messages": {} }, "pvc-7a8ded68-862d-4abb-a08c-8cf9664dab10": { "Name": "pvc-7a8ded68-862d-4abb-a08c-8cf9664dab10", "Size": "10737418240", "Labels": {}, "Created": "2021-05-10T02:43:02Z", "LastBackupName": "backup-432f4d6afa31481f", "LastBackupAt": "2021-05-10T06:04:02Z", "DataStored": "140509184", "Messages": {} } } ``` - `backup ls --volume `: List all backups and read it's config (`backup_backup_.cfg`). For example: ```json $ backup ls s3://backupbucket@minio/ --volume pvc-004d8edb-3a8c-4596-a659-3d00122d3f07 { "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": { "Name": "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "Size": "2147483648", "Labels": {}, "Created": "2021-05-12T00:52:01Z", "LastBackupName": "backup-c5f548b7e86b4b56", "LastBackupAt": "2021-05-17T05:31:01Z", "DataStored": "121634816", "Messages": {}, "Backups": { "s3://backupbucket@minio/?backup=backup-02224cb26b794e73\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": { "Name": "backup-02224cb26b794e73", "URL": "s3://backupbucket@minio/?backup=backup-02224cb26b794e73\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "SnapshotName": "backup-23c4fd9a", "SnapshotCreated": "2021-05-17T05:23:01Z", "Created": "2021-05-17T05:23:04Z", "Size": "115343360", "Labels": {}, "IsIncremental": true, "Messages": null }, ... "s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": { "Name": "backup-fa78d89827664840", "URL": "s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "SnapshotName": "backup-ac364071", "SnapshotCreated": "2021-05-17T04:42:01Z", "Created": "2021-05-17T04:42:03Z", "Size": "115343360", "Labels": {}, "IsIncremental": true, "Messages": null } } } } ``` - `backup inspect `: Read a single backup config (`backup_backup_.cfg`). For example: ```json $ backup inspect s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07 { "Name": "backup-fa78d89827664840", "URL": "s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "SnapshotName": "backup-ac364071", "SnapshotCreated": "2021-05-17T04:42:01Z", "Created": "2021-05-17T04:42:03Z", "Size": "115343360", "Labels": {}, "IsIncremental": true, "VolumeName": "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "VolumeSize": "2147483648", "VolumeCreated": "2021-05-12T00:52:01Z", "Messages": null } ``` After this enhancement, the [longhorn/backupstore](https://github.com/longhorn/backupstore) list and inspect command behavior are: - `backup ls --volume-only`: List all backup volume names. For example: ```json $ backup ls s3://backupbucket@minio/ --volume-only { "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": {}, "pvc-7a8ded68-862d-4abb-a08c-8cf9664dab10": {} } ``` - `backup ls --volume `: List all backup names. For example: ```json $ backup ls s3://backupbucket@minio/ --volume pvc-004d8edb-3a8c-4596-a659-3d00122d3f07 { "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07": { "Backups": { "backup-02224cb26b794e73": {}, ... "backup-fa78d89827664840": {} } } } ``` - `backup inspect-volume `: Read a single backup volume config (`volume.cfg`). For example: ```json $ backup inspect-volume s3://backupbucket@minio/?volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07 { "Name": "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "Size": "2147483648", "Labels": {}, "Created": "2021-05-12T00:52:01Z", "LastBackupName": "backup-c5f548b7e86b4b56", "LastBackupAt": "2021-05-17T05:31:01Z", "DataStored": "121634816", "Messages": {} } ``` - `backup inspect `: Read a single backup config (`backup_backup_.cfg`). For example: ```json $ backup inspect s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07 { "Name": "backup-fa78d89827664840", "URL": "s3://backupbucket@minio/?backup=backup-fa78d89827664840\u0026volume=pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "SnapshotName": "backup-ac364071", "SnapshotCreated": "2021-05-17T04:42:01Z", "Created": "2021-05-17T04:42:03Z", "Size": "115343360", "Labels": {}, "IsIncremental": true, "VolumeName": "pvc-004d8edb-3a8c-4596-a659-3d00122d3f07", "VolumeSize": "2147483648", "VolumeCreated": "2021-05-12T00:52:01Z", "Messages": null } ``` - `backup head `: Get the config metadata. For example: ```json { "ModificationTime": "2021-05-17T04:42:03Z", } ``` Generally speaking, we want to separate the **list**, **read**, and **head** commands. 2. The Longhorn manager HTTP endpoints. | HTTP Endpoint | Before | After | | ------------------------------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | | **GET** `/v1/backupvolumes` | read all backup volumes from the remote backup target | read all the BackupVolume CRs | | **GET** `/v1/backupvolumes/{volName}` | read a backup volume from the remote backup target | read a BackupVolume CR with the given volume name | | **DELETE** `/v1/backupvolumes/{volName}` | delete a backup volume from the remote backup target | delete the BackupVolume CR with the given volume name, `backup_volume_controller` reconciles to delete a backup volume from the remote backup target | | **POST** `/v1/volumes/{volName}?action=snapshotBackup` | create a backup to the remote backup target | create a new Backup, `backup_controller` reconciles to create a backup to the remote backup target | | **GET** `/v1/backupvolumes/{volName}?action=backupList` | read a list of backups from the remote backup target | read a list of Backup CRs with the label filter `volume=` | | **GET** `/v1/backupvolumes/{volName}?action=backupGet` | read a backup from the remote backup target | read a Backup CR with the given backup name | | **DELETE** `/v1/backupvolumes/{volName}?action=backupDelete` | delete a backup from the remote backup target | delete the Backup CR, `backup_controller` reconciles to delete a backup from reomte backup target | ## Design ### Implementation Overview 1. Create a new BackupTarget CRD `backuptargets.longhorn.io`. ```yaml metadata: name: the backup target name (`default`), since currently we only support one backup target. spec: backupTargetURL: the backup target URL. (string) credentialSecret: the backup target credential secret. (string) pollInterval: the backup target poll interval. (metav1.Duration) syncRequestAt: the time to request run sync the remote backup target. (*metav1.Time) status: ownerID: the node ID which is responsible for running operations of the backup target controller. (string) available: records if the remote backup target is available or not. (bool) lastSyncedAt: records the last time the backup target was running the reconcile process. (*metav1.Time) ``` 2. Create a new BackupVolume CRD `backupvolumes.longhorn.io`. ```yaml metadata: name: the backup volume name. spec: syncRequestAt: the time to request run sync the remote backup volume. (*metav1.Time) fileCleanupRequired: indicate to delete the remote backup volume config or not. (bool) status: ownerID: the node ID which is responsible for running operations of the backup volume controller. (string) lastModificationTime: the backup volume config last modification time. (Time) size: the backup volume size. (string) labels: the backup volume labels. (map[string]string) createAt: the backup volume creation time. (string) lastBackupName: the latest volume backup name. (string) lastBackupAt: the latest volume backup time. (string) dataStored: the backup volume block count. (string) messages: the error messages when call longhorn engine on list or inspect backup volumes. (map[string]string) lastSyncedAt: records the last time the backup volume was synced into the cluster. (*metav1.Time) ``` 3. Create a new Backup CRD `backups.longhorn.io`. ```yaml metadata: name: the backup name. labels: longhornvolume=`: this label indicates which backup volume the backup belongs to. spec: fileCleanupRequired: indicate to delete the remote backup config or not (and the related block files if needed). (bool) snapshotName: the snapshot name. (string) labels: the labels of snapshot backup. (map[string]string) backingImage: the backing image. (string) backingImageURL: the backing image URL. (string) status: ownerID: the node ID which is responsible for running operations of the backup controller. (string) backupCreationIsStart: to indicate the snapshot backup creation is start or not. (bool) url: the snapshot backup URL. (string) snapshotName: the snapshot name. (string) snapshotCreateAt: the snapshot creation time. (string) backupCreateAt: the snapshot backup creation time. (string) size: the snapshot size. (string) labels: the labels of snapshot backup. (map[string]string) messages: the error messages when calling longhorn engine on listing or inspecting backups. (map[string]string) lastSyncedAt: records the last time the backup was synced into the cluster. (*metav1.Time) ``` 4. At the existed `setting_controller`. Watches the changes of Setting CR `settings.longorn.io` field `backup-target`, `backup-target-credential-secret`, and `backupstore-poll-interval`. The setting controller is responsible for creating/updating the default BackupTarget CR. The setting controller creates a timer goroutine according to `backupstore-poll-interval`. Once the timer up, updates the BackupTarget CR `spec.syncRequestAt = time.Now()`. If the `backupstore-poll-interval = 0`, do not updates the BackupTarget CR `spec.syncRequestAt`. 5. Create a new `backup_target_controller`. Watches the change of BackupTarget CR. The backup target controller is responsible for creating/updating/deleting BackupVolume CR metadata+spec. The reconcile loop steps are: 1. Check if the current node ID == BackupTarget CR spec.responsibleNodeID. If no, skip the reconcile process. 2. Check if the `status.lastSyncedAt < spec.syncRequestAt`. If no, skip the reconcile process. 3. Call the longhorn engine to list all the backup volumes `backup ls --volume-only` from the remote backup target `backupStoreBackupVolumes`. If the remote backup target is unavailable: 1. Updates the BackupTarget CR `status.available=false` and `status.lastSyncedAt=time.Now()`. 2. Skip the current reconcile process. 4. List in cluster BackupVolume CRs `clusterBackupVolumes`. 5. Find the difference backup volumes `backupVolumesToPull = backupStoreBackupVolumes - clusterBackupVolumes` and create BackupVolume CR `metadata.name`. 6. Find the difference backup volumes `backupVolumesToDelete = clusterBackupVolumes - backupStoreBackupVolumes` and delete BackupVolume CR. 7. List in cluster BackupVolume CRs `clusterBackupVolumes` again and updates the BackupVolume CR `spec.syncRequestAt = time.Now()`. 8. Updates the BackupTarget CR status: 1. `status.available=true`. 2. `status.lastSyncedAt = time.Now()`. 6. For the Longhorn manager HTTP endpoints: - **DELETE** `/v1/backupvolumes/{volName}`: 1. Update the BackupVolume CR `spec.fileCleanupRequired=true` with the given volume name. 2. Delete a BackupVolume CR with the given volume name. 7. Create a new controller `backup_volume_controller`. Watches the change of BackupVolume CR. The backup volume controller is responsible for deleting Backup CR and deleting backup volume from remote backup target if delete BackupVolume CR event comes in, and updating BackupVolume CR status field, and creating/deleting Backup CR. The reconcile loop steps are: 1. Check if the current node ID == BackupTarget CR spec.responsibleNodeID. If no, skip the reconcile process. 2. If the delete BackupVolume CR event comes in: 1. updates Backup CRs `spec.fileCleanupRequired=true` if BackupVolume CR `spec.fileCleanupRequired=true`. 2. deletes Backup CR with the given volume name. 3. deletes the backup volume from the remote backup target `backup rm --volume ` if `spec.fileCleanupRequired=true`. 4. remove the finalizer. 3. Check if the `status.lastSyncedAt < spec.syncRequestAt`. If no, skip the reconcile process. 4. Call the longhorn engine to list all the backups `backup ls --volume ` from the remote backup target `backupStoreBackups`. 5. List in cluster Backup CRs `clusterBackups`. 6. Find the difference backups `backupsToPull = backupStoreBackups - clusterBackups` and create Backup CR `metadata.name` + `metadata.labels["longhornvolume"]=`. 7. Find the difference backups `backupsToDelete = clusterBackups - backupStoreBackups` and delete Backup CR. 8. Call the longhorn engine to get the backup volume config's last modification time `backup head ` and compares to `status.lastModificationTime`. If the config last modification time not changed, updates the `status.lastSyncedAt` and return. 9. Call the longhorn engine to read the backup volumes' config `backup inspect-volume `. 10. Updates the BackupVolume CR status: 1. according to the backup volumes' config. 2. `status.lastModificationTime` and `status.lastSyncedAt`. 11. Updates the Volume CR `status.lastBackup` and `status.lastBackupAt`. 8. For the Longhorn manager HTTP endpoints: - **POST** `/v1/volumes/{volName}?action=snapshotBackup`: 1. Generate the backup name . 2. Create a new Backup CR with ```yaml metadata: name: labels: longhornvolume: spec: snapshotName: labels: backingImage: backingImageURL: ``` - **DELETE** `/v1/backupvolumes/{volName}?action=backupDelete`: 1. Update the Backup CR `spec.fileCleanupRequired=true` with the given volume name. 2. Delete a Backup CR with the given backup name. 9. Create a new controller `backup_controller`. Watches the change of Backup CR. The backup controller is responsible for updating the Backup CR status field and creating/deleting backup to/from the remote backup target. The reconcile loop steps are: 1. Check if the current node ID == BackupTarget CR spec.responsibleNodeID. If no, skip the reconcile process. 2. If the delete Backup CR event comes in: 1. delete the backup from the remote backup target `backup rm ` if Backup CR `spec.fileCleanupRequired=true`. 2. update the BackupVolume CR `spec.syncRequestAt=time.Now()`. 3. remove the finalizer. 3. Check if the Backup CR `spec.snapshotName != ""` and `status.backupCreationIsStart == false`. If yes: 1. call longhorn engine/replica for backup creation. 2. updates Backup CR `status.backupCreationIsStart = true`. 3. fork a go routine to monitor the backup creation progress. After backup creation finished (progress = 100): 1. update the BackupVolume CR `spec.syncRequestAt = time.Now()` if BackupVolume CR exist. 2. create the BackupVolume CR `metadata.name` if BackupVolume CR not exist. 4. If Backup CR `status.lastSyncedAt != nil`, the backup config had be synced, skip the reconcile process. 5. Call the longhorn engine to read the backup config `backup inspect `. 6. Updates the Backup CR status field according to the backup config. 7. Updates the Backup CR `status.lastSyncedAt`. 10. For the Longhorn manager HTTP endpoints: - **GET** `/v1/backupvolumes`: read all the BackupVolume CRs. - **GET** `/v1/backupvolumes/{volName}`: read a BackupVolume CR with the given volume name. - **GET** `/v1/backupvolumes/{volName}?action=backupList`: read a list of Backup CRs with the label filter `volume=`. - **GET** `/v1/backupvolumes/{volName}?action=backupGet`: read a Backup CR with the given backup name. ### Test plan With over 1k backup volumes and over 1k backups under pretty high network latency (700-800ms per operation) from longhorn manager to the remote backup target: - Test basic backup and restore operations. 1. The user configures the remote backup target URL/credential and poll interval to 5 mins. 2. The user creates two backups on vol-A and vol-B. 3. The user can see the backup volume for vol-A and vol-B in Backup GUI. 4. The user can see the two backups under vol-A and vol-B in Backup GUI. 5. When the user deletes one of the backups of vol-A on the Longhorn GUI, the deleted one will be deleted after the remote backup target backup be deleted. 6. When the user deletes backup volume vol-A on the Longhorn GUI, the backup volume will be deleted after the remote backup target backup volume is deleted, and the backup of vol-A will be deleted also. 7. The user can see the backup volume for vol-B in Backup GUI. 8. The user can see two backups under vol-B in Backup GUI. 9. The user changes the remote backup target to another backup target URL/credential, the user can't see the backup volume and backup of vol-B in Backup GUI. 10. The user configures the `backstore-poll-interval` to 1 minute. 11. The user changes the remote backup target to the original backup target URL/credential, after 1 minute later, the user can see the backup volume and backup of vol-B. 12. Create volume from the vol-B backup. - Test DR volume operations. 1. Create two clusters (cluster-A and cluster-B) both points to the same remote backup target. 2. At cluster A, create a volume and run a recurring backup to the remote backup target. 3. At cluster B, after `backupstore-poll-interval` seconds, the user can list backup volumes or list volume backups on the Longhorn GUI. 4. At cluster B, create a DR volume from the backup volume. 5. At cluster B, check the DR volume `status.LastBackup` and `status.LastBackupAt` is updated periodically. 6. At cluster A, delete the backup volume on the GUI. 7. At cluster B, after `backupstore-poll-interval` seconds, the deleted backup volume does not exist on the Longhorn GUI. 8. At cluster B, the DR volume `status.LastBackup` and `status.LastBackupAt` won't be updated anymore. - Test Backup Target URL clean up. 1. The user configures the remote backup target URL/credential and poll interval to 5 mins. 2. The user creates one backup on vol-A. 3. Change the backup target setting setting to empty. 4. Within 5 mins the poll interval triggered: 1. The default BackupTarget CR `status.available=false`. 2. The default BackupTarget CR `status.lastSyncedAt` be updated. 3. All the BackupVolume CRs be deleted. 4. All the Backup CRs be deleted. 5. The vol-A CR `status.lastBackup` and `status.lastBackupAt` be cleaned up. 5. The GUI displays the backup target not available. - Test switch Backup Target URL. 1. The user configures the remote backup target URL/credential to S3 and poll interval to 5 mins. 2. The user creates one backup on vol-A to S3. 3. The user changes the remote backup URL/credential to NFS and poll interval to 5 mins. 4. The user creates one backup on vol-A to NFS. 5. The user changes the remote backup target URL/credential to S3. 6. Within 5 mins the poll interval triggered: 1. The default BackupTarget CR `status.available=true`. 2. The default BackupTarget CR `status.lastSyncedAt` be updated. 3. The BackupVolume CRs be synced as the data in S3. 4. The Backup CRs be synced as the data in S3. 5. The vol-A CR `status.lastBackup` and `status.lastBackupAt` be synced as the data in S3. - Test Backup Target credential secret changed. 1. The user configures the remote backup target URL/credential and poll interval to 5 mins. 2. The user creates one backup on vol-A. 3. Change the backup target credential secret setting to empty. 4. Within 5 mins the poll interval triggered: 1. The default BackupTarget CR `status.available=false`. 2. The default BackupTarget CR `status.lastSyncedAt` be updated. 5. The GUI displays the backup target not available. ### Upgrade strategy None. ## Note With this enhancement, the user might want to trigger run synchronization immediately. We could either: - have a button on the `Backup` to update the BackupTarget CR `spec.syncRequestAt = time.Now()` _or_ have a button on the `Backup` -> `Backup Volume` page to have a button to update the BackupVolume CR `spec.syncRequestAt = time.Now()`. - updates the `spec.syncRequestAt = time.Now()` when the user clicks the `Backup` _or_ updates the `spec.syncRequestAt = time.Now()` when the user clicks `Backup` -> `Backup Volume`. ================================================ FILE: enhancements/20210624-label-driven-recurring-job.md ================================================ # Label-driven Recurring Job ## Summary Replace volume spec `recurringJobs` with the label-driven model. Abstract volume recurring jobs to a new CRD named "RecurringJob". The names or groups of recurring jobs can be referenced in volume labels. Users can set a recurring job to the `Default` group and Longhorn will automatically apply when the volume has no job labels. Only one cron job will be created per recurring job. Users can also update the recurring job, and Longhorn reflects the changes to the associated cron job. When instruct Longhorn to delete a recurring job, this will also remove the associated cron job. StorageClass now should use `recurringJobSelector` instead to refer to the recurring job names. During the version upgrade, existing volume spec `recurringJobs` and storageClass `recurringJobs` will automatically translate to volume labels, and recurring job CRs will get created. ### Related Issues https://github.com/longhorn/longhorn/issues/467 ## Motivation ### Goals Phase 1: - Each recurring job can be in single or multiple groups. - Each group can have single or multiple recurring jobs. - The jobs and groups can be reference with the volume label. - The recurring job in `default` group will automatically apply to a volume that has no job labels. - Can create multiple recurring jobs with UI or YAML. - The recurring job should include settings; `name`, `groups`, `task`, `cron`, `retain`, `concurrency`, `labels`. Phase2: The StorageClass and upgrade migration are still dependent on volume spec, thus complete removal of volume spec should be done in phase 2. ### Non-goals [optional] 1. Does the snapshot/backup operation one by one. The operation order can be defined as sequential, consistent (for volume group snapshot), or throttled (with a concurrent number as a parameter) in the future. > https://github.com/longhorn/longhorn/pull/2737#issuecomment-887985811 ## Proposal #### Story 1 - set recurring jobs and groups by volume labels As a Longhorn user / System administrator. I want to directly update recurring jobs referenced in multiple volumes. So I do not need to update each volume with cron job definition. #### Story 2 - automatically applies for the default recurring jobs. As a Longhorn user / System administrator. I want the ability to set one or multiple `backup` and `snapshot` recurring jobs as default. All volumes without any recurring job label should automatically apply with the default recurring jobs. So I can be assured that all volumes without any recurring job label will automatically apply with default. #### Story 3 - automatically upgrade migration As a Longhorn user / System administrator I want Longhorn to automatically convert existing volume spec `recurringJobs` to volume labels, and create associate recurring job CRs. So I don't have to manually create recurring job CRs and patch the labels. ### User Experience In Detail #### Story 1 - set recurring job in volume 1. Create a recurring job on UI `Recurring Job` page or via `kubectl`. 2. In UI, Navigate to Volume, `Recurring Jobs Schedule`. 3. User can choose from `Job` or job `Group` from the tab. - On `Job` tab, 1. User sees existing recurring jobs that volume had labeled. 1. User able to select `Backup` or `Snapshot` for the `Type` from the drop-down list. 2. User able to edit the `Schedule`, `Retain`, `Concurrency` and `Labels`. - On the job `Group` tab. 1. User sees all existing recurring job groups from the `Name` drop-down list. 2. User selects the job from the drop-down list. 3. User sees all recurring `jobs` under the `group`. 4. Click `Save` updates to the volume label. 5. Update the recurring job CRs also reflect on the cron job and UI `Recurring Jobs Schedule`. **Before enhancement** Recurring jobs can only be added and updated per volume spec. Create cron job definitions for each volume causing duplicated setup effort. The recurring job can only be updated per volume. **After enhancement** Recurring jobs can be added and update as the volume label. Can select a recurring job from the UI drop-down menu and will automatically show the information from the recurring job CRs. Update the recurring job definition will automatically apply to all volumes with the job label. #### Story 2 - automatically apply to default recurring jobs 1. Add `default` to one or multiple recurring jobs `Groups` in UI or `kubectl`. 2. Longhorn automatically applies the `default` group recurring jobs to all volumes without job labels. **Before enhancement** Default recurring jobs are set via StorageClass at PVC creation only. No default recurring job can be set up for UI-created volumes. Updating StorageClass does not reflect on the existing volumes. **After enhancement** Have the option to set default recurring jobs via `StorageClass` or `RecurringJob`. Longhorn recurring job controller automatically applies default recurring jobs to all volumes without the job labels. Longhorn adds the default recurring jobs when all job labels are removed from the volume. When the `RecurringJobSelector` is set in the `StorageClass`, it will be used as default instead. #### Story 3 - automatically upgrade migration 1. Perform upgrade. 2. StorageClass `recurringJobs` will get convert to `recurringJobSelector`. 3. Recurring job CRs will get created from `recurringJobs`. 4. Volume will be labeled with `recurring-job.longhorn.io/---: enabled` from volume spec `recurringJobs`. 5. Recurring job CRs will get created from volume spec `recurringJobs`. When the config is identical among multiple volumes, only one will get created and volumes will share this recurring job CR. 6. Volume spec `recurringJobs` will get removed. ### API Changes - Add new HTTP endpoints: - GET `/v1/recurringjobs` to list of recurring jobs. - GET `/v1/recurringjobs/{name}` to get specific recurring job. - DELETE `/v1/recurringjobs/{name}` to delete specific recurring job. - POST `/v1/recurringjobs` to create a recurring job. - PUT `/v1/recurringjobs/{name}` to update specific recurring job. - `/v1/ws/recurringjobs` and `/v1/ws/{period}/recurringjobs` for websocket stream. - Add new RESTful APIs for the new `RecurringJob` CRD: - `Create` - `Update` - `List` - `Get` - `Delete` - Add new APIs for users to update recurring jobs for individual volume. The : - `/v1/volumes/?action=recurringJobAdd`, expect request's body in form {name:, isGroup:}. - `/v1/volumes/?action=recurringJobList`. - `/v1/volumes/?action=recurringJobDelete`, expect request's body in form {name:, isGroup:}. ## Design ### Implementation Overview #### Add Recurring Job CRD. - Update the ClusterRole to include `recurringjob`. - Printer column should include `Name`, `Groups`, `Task`, `Cron`, `Retain`, `Concurrency`, `Age`, `Labels`. ``` NAME GROUPS TASK CRON RETAIN CONCURRENCY AGE LABELS snapshot1 ["default","group1"] snapshot * * * * * 1 2 14m {"label/1":"a","label/2":"b"} ``` - The `Name`: String used to reference the recurring job in volume with the label `recurring-job.longhorn.io/: enabled`. - The `Groups`: Array of strings that set groupings to the recurring job. This is used to reference the recurring job group in volume with the label `recurring-job-group.longhorn.io/: enabled`. When including `default`, the recurring job will be added to the volume label if no other job exists in the volume label. - The `Task`: String of either one of `backup` or `snapshot`. Also, add validation in the CRD YAML with pattern regex match. - The `Cron`: String in cron expression represents recurring job scheduling. - The `Retain`: Integer of the number of snapshots/backups to keep for the volume. - The `Concurrency`: Integer of the concurrent job to run by each cron job. - The `Age`: Date of the CR creation timestamp. - The `Labels`: Dictionary of the labels. #### Add Command `recurring-job` To `longhorn-manager` Binary 1. Add new command `recurring-job --longhorn-manager ` and remove old command `snapshot`. > Get the `recurringJob.Spec` on execution using Kubernetes API. 2. Get volumes by label selector `recurring-job.longhorn.io/: enabled` to filter out volumes. 3. Get volumes by label selector `recurring-job-group.longhorn.io/: enabled` to filter out volumes if the job is set with a group. 4. Filter and create a list of the volumes in the state `attached` or setting `allow-recurring-job-while-volume-detached`. 5. Use the concurrent number parameter to throttle goroutine with channel. Each goroutine creates `NewJob()` and `job.run()` for the volumes. The job `snapshotName` format will be `-c-`. #### Changes In The Volume Controller 1. The `updateRecurringJobs` method is responsible to add the default label if not other labels exist. > Since the storage class and upgrade migration contains recurringJobs spec. So we will keep the `VolumeSpec.RecurringJobs` in code to create the recurring jobs for volumes from the `storageClass`. > In case names are duplicated between different `storageClasses`, only one recurring job CR will be created. #### Changes In The VolumeManager `CreateVolume` - Add new method input `recurringJobSelector`: 1. Convert `Volume.Spec.RecurringJobs` to `recurringJobSelector`. 2. Add recurring job label if `recurringJobSelector` method input is not empty. #### Changes In The Datastore - For `CreateVolume` and `UpdateVolume` add a function similar to `fixupMetadata` that handles recurring jobs: 1. Add recurring job labels if `Volume.Spec.RecurringJobs` is not empty. Then unset `Volume.Spec.RecurringJobs`. 2. Label with `default` job-group if no other recurring job label exists. #### Introduce `recurringJobSelector` As Part Of StorageClass Parameters. - The CSI controller can use `recurringJobSelector` for volume creation. #### Changes In CSI Controller Server 1. Put `recurringJobSelector` to `vol.RecurringJobSelector` at HTTP API layer to use for adding volume recurring job label in `VolumeManager.CreateVolume`. The `CreateVolume` method will have a new input `recurringJobSelector`. 2. Get `recurringJobs` from parameters, validate and create recurring job CRs via API if not already exist. #### Add Recurring Job Controller - The code structure will be the same as other controllers. - Add the finalizer to the recurring job CRs if not exist. - The controller will be informed by `recurringJobInformer` and `enqueueRecurringJob`. - Create and update CronJob per recurring job. 1. Generate a new cron job object. - Include labels `recurring-job.longhorn.io`. ``` recurring-job.longhorn.io: ``` - Compose command, ``` longhorn-manager -d\ recurring-job \ --manager-url ``` 2. Create new cron job with annotation `last-applied-cronjob-spec` or update cron job if the new cron job spec is different from the `last-applied-cronjob-spec`. - Use defer to clean up CronJob. 1. When a recurring job gets deleted. 1. Delete the cron job with selected labels: `recurring-job.longhorn.io/`. 2. Remove the finalizer. #### UI ##### Add New Page `Recurring Job` In UI A new page for `Recurring Job` to create/update/delete recurring jobs. ``` Recurring Job [Custom Column] ==================================================================================================================== [Create] [Delete] [Search Box v ][__________][Go] | Name | Group | Type | Schedule | Labels | Retain | Concurrency =================================================================================================================== [] | Name | Group | Type | Schedule | Labels | Retain | Concurrency | Operation | ---+-------+--------+--------+-----------------+--------------+--------+-------------+-------------+--------------| [] | dummy | aa, bb | backup | 00:00 every day | k1:v1, k2:v2 | 20 | 10 | [Icon] v | | Update | Delete =================================================================================================================== [<] [1] [>] ``` **Scenario: Add Recurring Job** *Given* user sees `Create` on top left of the page. *When* user click `Create`. *Then* user sees a pop-up form. ``` * Name [ ] Groups + * Task [Backup] * Schedule [00:00 every day] * Retain [20] * Concurrency [10] * Labels + ``` - Field with `*` is mendatory - User can click on `+` next to `Group` to add more groups. - User can click on the `Schedule` field and a window will pop-up for `Cron` and `Generate Cron`. - `Retain` cannot be `0`. - `Concurrency` cannot be `0`. - User can click on `+` next to `Labels` to add more labels. *When* user click `OK`. *Then* frontend **POST** `/v1/recurringjobs` to create a recurring job. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "sample", "groups": ["group-1", "group-2"], "task": "snapshot", "cron": "* * * * *", "retain": 2, "concurrency": 1, "labels": {"label/1": "a"}}' \ http://54.251.150.85:30944/v1/recurringjobs | jq { "actions": {}, "concurrency": 1, "cron": "* * * * *", "groups": [ "group-1", "group-2" ], "id": "sample", "labels": { "label/1": "a" }, "links": { "self": "http://54.251.150.85:30944/v1/recurringjobs/sample" }, "name": "sample", "retain": 2, "task": "snapshot", "type": "recurringJob" } ``` **Scenario: Update Recurring Job** *Given* an `Operation` drop-down list next to the recurring job. *When* user click `Edit`. *Then* user sees a pop-up form. ``` Name [sample] Groups [group-1] [group-2] Task [Backup] Schedule [00:00 every day] Retain [20] Concurrency [10] Labels [labels/1]: [a] [labels/2]: [b] ``` - `Name` field should be immutable. - `Task` field should be immutable. *And* user edit the fields in the form. *When* user click `Save`. *Then* frontend **PUT** `/v1/recurringjobs/{name}` to update specific recurring job. ``` ❯ curl -X PUT -H "Content-Type: application/json" \ -d '{"name": "sample", "groups": ["group-1", "group-2"], "task": "snapshot", "cron": "* * * * *", "retain": 2, "concurrency": 1, "labels": {"label/1": "a", "label/2": "b"}}' \ http://54.251.150.85:30944/v1/recurringjobs/sample | jq { "actions": {}, "concurrency": 1, "cron": "* * * * *", "groups": [ "group-1", "group-2" ], "id": "sample", "labels": { "label/1": "a", "label/2": "b" }, "links": { "self": "http://54.251.150.85:30944/v1/recurringjobs/sample" }, "name": "sample", "retain": 2, "task": "snapshot", "type": "recurringJob" } ``` **Scenario: Delete Recurring Job** *Given* an `Operation` drop-down list next to the recurring job. *When* user click `Delete`. *Then* user should see a pop-up window for confirmation. *When* user click `OK`. *Then* frontend **DELETE** `/v1/recurringjobs/{name}` to delete specific recurring job. ``` ❯ curl -X DELETE http://54.251.150.85:30944/v1/recurringjobs/sample | jq ``` > Also need a button for batch deletion on top left of the table. ##### Updates `Volume` Page On UI **Scenario: Select From Recurring Job or Job Group** *When* user should be able to choose if want to add recurring job as `Job` or `Group` from the tab. **Scenario: Add Recurring Job Group On Volume Page** *Given* user go to job `Group` tab. *When* user click `+ New`. *And* Frontend can **GET** `/v1/recurringjobs` to list of recurring jobs. *And* Frontend need to gather all `groups` from data. ``` ❯ curl -X GET http://54.251.150.85:30783/v1/recurringjobs | jq { "data": [ { "actions": {}, "concurrency": 2, "cron": "* * * * *", "groups": [ "group2", "group3" ], "id": "backup1", "labels": null, "links": { "self": "http://54.251.150.85:30783/v1/recurringjobs/backup1" }, "name": "backup1", "retain": 1, "task": "backup", "type": "recurringJob" }, { "actions": {}, "concurrency": 2, "cron": "* * * * *", "groups": [ "default", "group1" ], "id": "snapshot1", "labels": { "label/1": "a", "label/2": "b" }, "links": { "self": "http://54.251.150.85:30783/v1/recurringjobs/snapshot1" }, "name": "snapshot1", "retain": 1, "task": "snapshot", "type": "recurringJob" } ], "links": { "self": "http://54.251.150.85:30783/v1/recurringjobs" }, "resourceType": "recurringJob", "type": "collection" } ``` *Then* the user selects the group from the drop-down list. *When* user click on `Save`. *Then* frontend **POST** `/v1/volumes/?action=recurringJobAdd` with request body `{name: , isGroup: true}`. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "test3", "isGroup": true}' \ http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63\?action\=recurringJobAdd | jq { "data": [ { "actions": {}, "id": "default", "isGroup": true, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/default" }, "name": "default", "type": "volumeRecurringJob" }, { "actions": {}, "id": "test3", "isGroup": true, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/test3" }, "name": "test3", "type": "volumeRecurringJob" } ], "links": { "self": "http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63" }, "resourceType": "volumeRecurringJob", "type": "collection" } ``` *And* user sees all `jobs` with the `group`. **Scenario: Remove Recurring Job Group On Volume Page** *Given* user go to job `Group` tab. *When* user click the `bin` icon of the recurring job group. *Then* frontend `/v1/volumes/?action=recurringJobDelete` with request body `{name: , isGroup: true}`. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "test3", "isGroup": true}' \ http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63\?action\=recurringJobDelete | jq { "data": [ { "actions": {}, "id": "default", "isGroup": true, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/default" }, "name": "default", "type": "volumeRecurringJob" } ], "links": { "self": "http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63" }, "resourceType": "volumeRecurringJob", "type": "collection" } ``` **Scenario: Add Recurring Job On Volume Page** *Given* user go to `Job` tab. *When* user click `+ New`. *And* user sees the name is auto-generated. *And* user can select `Backup` or `Snapshot` from the drop-down list. *And* user can edit `Schedule`, `Labels`, `Retain` and `Concurrency`. *When* user click on `Save`. *Then* frontend **POST** /v1/recurringjobs to create a recurring job. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "backup1", "groups": [], "task": "backup", "cron": "* * * * *", "retain": 2, "concurrency": 1, "labels": {"label/1": "a"}}' \ http://54.251.150.85:30944/v1/recurringjobs | jq { "actions": {}, "concurrency": 1, "cron": "* * * * *", "groups": [], "id": "backup1", "labels": { "label/1": "a" }, "links": { "self": "http://54.251.150.85:30783/v1/recurringjobs/backup1" }, "name": "backup1", "retain": 2, "task": "backup", "type": "recurringJob" } ``` *And* frontend **POST** `/v1/volumes/?action=recurringJobAdd` with request body `{name: , isGroup: false}`. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "backup1", "isGroup": false}' \ http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63\?action\=recurringJobAdd | jq { "data": [ { "actions": {}, "id": "default", "isGroup": true, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/default" }, "name": "default", "type": "volumeRecurringJob" }, { "actions": {}, "id": "backup1", "isGroup": false, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/backup1" }, "name": "backup1", "type": "volumeRecurringJob" } ], "links": { "self": "http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63" }, "resourceType": "volumeRecurringJob", "type": "collection" } ``` **Scenario: Delete Recurring Job On Volume Page** Same as **Scenario: Remove Recurring Job Group in Volume Page** with request body `{name: , isGroup: false}`. ``` ❯ curl -X POST -H "Content-Type: application/json" \ -d '{"name": "backup1", "isGroup": false}' \ http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63\?action\=recurringJobDelete | jq { "data": [ { "actions": {}, "id": "default", "isGroup": true, "links": { "self": "http://54.251.150.85:30783/v1/volumerecurringjobs/default" }, "name": "default", "type": "volumeRecurringJob" } ], "links": { "self": "http://54.251.150.85:30783/v1/volumes/pvc-4011f9a6-bae3-43e3-a2a1-893997d0aa63" }, "resourceType": "volumeRecurringJob", "type": "collection" } ``` **Scenario: Keep Recurring Job Details Updated On Volume Page** - Frontend can monitor new websocket `/v1/ws/recurringjobs` and `/v1/ws/{period}/recurringjobs`. - When a volume is labeled with a none-existing recurring job or job-group. UI should show warning icon. ### Test plan > The existing recurring job test cases need to be fixed or replaced. #### Integration test - test_recurring_job_group Scenario: test recurring job groups (S3/NFS) Given create `snapshot1` recurring job with `group-1, group-2` in groups. set cron job to run every 2 minutes. set retain to 1. create `backup1` recurring job with `group-1` in groups. set cron job to run every 3 minutes. set retain to 1 And volume `test-job-1` created, attached, and healthy. volume `test-job-2` created, attached, and healthy. When set `group1` recurring job in volume `test-job-1` label. set `group2` recurring job in volume `test-job-2` label. And write some data to volume `test-job-1`. write some data to volume `test-job-2`. And wait for 2 minutes. And write some data to volume `test-job-1`. write some data to volume `test-job-2`. And wait for 1 minute. Then volume `test-job-1` should have 3 snapshots after scheduled time. volume `test-job-2` should have 2 snapshots after scheduled time. And volume `test-job-1` should have 1 backup after scheduled time. volume `test-job-2` should have 0 backup after scheduled time. #### Integration test - test_recurring_job_default Scenario: test recurring job set with default in groups Given 1 volume created, attached, and healthy. When create `snapshot1` recurring job with `default, group-1` in groups. create `snapshot2` recurring job with `default` in groups.. create `snapshot3` recurring job with `` in groups. create `backup1` recurring job with `default, group-1` in groups. create `backup2` recurring job with `default` in groups. create `backup3` recurring job with `` in groups. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. `snapshot3` cron job should not exist. default `backup1` cron job should exist. default `backup2` cron job should exist. `backup3` cron job should not exist. # Setting recurring job in volume label should not remove the defaults. When set `snapshot3` recurring job in volume label. Then should contain `default` job-group in volume labels. should contain `snapshot3` job in volume labels. And default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. `snapshot3` cron job should exist. default `backup1` cron job should exist. default `backup2` cron job should exist. `backup3` cron job should not exist. # Should be able to remove the default. When delete recurring job-group `default` in volume label. And default `snapshot1` cron job should not exist. default `snapshot2` cron job should not exist. `snapshot3` cron job should exist. default `backup1` cron job should not exist. default `backup2` cron job should not exist. `backup3` cron job should not exist. # Remove all volume recurring job labels should bring in default When delete all recurring jobs in volume label. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. `snapshot3` cron job should not exist. default `backup1` cron job should exist. default `backup2` cron job should exist. `backup3` cron job should not exist. # Add `default` to snapshot3 and backup3 recurring job `Group`. # should also reflect on the cron jobs When add `snapshot3` recurring job with `default` in groups. add `backup3` recurring job with `default` in groups. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. default `snapshot3` cron job should exist. default `backup1` cron job should exist. default `backup2` cron job should exist. default `backup3` cron job should exist. # Remove `default` in recurring job `Group` should also # reflect on the cron jobs When remove `default` from `snapshot3` recurring job groups. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. `snapshot3` cron job should not exist. default `backup1` cron job should exist. default `backup2` cron job should exist. default `backup3` cron job should exist. # Remove `default` in all recurring job `Group` should also # reflect on the cron jobs When remove `default` from all recurring jobs groups. Then `snapshot1` cron job should not exist. `snapshot2` cron job should not exist. `snapshot3` cron job should not exist. `backup1` cron job should not exist. `backup2` cron job should not exist. `backup3` cron job should not exist. #### Integration test - test_recurring_job_delete Scenario: test delete recurring job Given 1 volume created, attached, and healthy. When create `snapshot1` recurring job with `default, group-1` in groups. create `snapshot2` recurring job with `default` in groups.. create `snapshot3` recurring job with `` in groups. create `backup1` recurring job with `default, group-1` in groups. create `backup2` recurring job with `default` in groups. create `backup3` recurring job with `` in groups. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should exist. `snapshot3` cron job should not exist. default `backup1` cron job should exist. default `backup2` cron job should exist. `backup3` cron job should not exist. # Delete `snapshot2` recurring job should delete the cron job When delete `snapshot-2` recurring job. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should not exist. `snapshot3` cron job should not exist. default `backup1` cron job should exist. default `backup2` cron job should exist. `backup3` cron job should not exist. # Delete multiple recurring jobs should reflect on the cron jobs. When delete `backup-1` recurring job. delete `backup-2` recurring job. delete `backup-3` recurring job. Then default `snapshot1` cron job should exist. default `snapshot2` cron job should not exist. `snapshot3` cron job should not exist. default `backup1` cron job should not exist. default `backup2` cron job should not exist. `backup3` cron job should not exist. # Should be able to delete recurring job while existing in volume label When add `snapshot1` recurring job to volume label. add `snapshot3` recurring job to volume label. And default `snapshot1` cron job should exist. default `snapshot2` cron job should not exist. `snapshot3` cron job should exist. And delete `snapshot1` recurring job. delete `snapshot3` recurring job. Then default `snapshot1` cron job should not exist. default `snapshot2` cron job should not exist. `snapshot3` cron job should not exist. #### Integration test - test_recurring_job_volume_labeled_none_existing_recurring_job Scenario: test volume with a none-existing recurring job label and later on added back. Given create `snapshot1` recurring job. create `backup1` recurring job. And 1 volume created, attached, and healthy. add `snapshot1` recurring job to volume label. add `backup1` recurring job to volume label. And `snapshot1` cron job exist. `backup1` cron job exist. When delete `snapshot1` recurring job. delete `backup1` recurring job. Then `snapshot1` cron job should not exist. `backup1` cron job should not exist. And `snapshot1` recurring job should exist in volume label. `backup1` recurring job should exist in volume label. # Add back the recurring jobs. When create `snapshot1` recurring job. create `backup1` recurring job. Then `snapshot1` cron job should exist. `backup1` cron job should exist. #### Integration test - test_recurring_job_with_multiple_volumes Scenario: test recurring job with multiple volumes Given volume `test-job-1` created, attached and healthy. And create `snapshot1` recurring job with `default` in groups. create `snapshot2` recurring job with `` in groups. create `backup1` recurring job with `default` in groups. create `backup2` recurring job with `` in groups. And volume `test-job-1` should have recurring job-group `default` label. And default `snapshot1` cron job exist. default `backup1` cron job exist. When create and attach volume `test-job-2`. wait for volume `test-job-2` to be healthy. Then volume `test-job-2` should have recurring job-group `default` label. When add `snapshot2` in `test-job-2` volume label. add `backup2` in `test-job-2` volume label. Then default `snapshot1` cron job should exist. `snapshot2` cron job should exist. default `backup1` cron job should exist. `backup2` cron job should exist. And volume `test-job-1` should have recurring job-group `default` label. volume `test-job-2` should have recurring job `snapshot2` label. volume `test-job-2` should have recurring job `backup2` label. #### Integration test - test_recurring_job_snapshot Scenario: test recurring job snapshot Given volume `test-job-1` created, attached, and healthy. volume `test-job-2` created, attached, and healthy. When create `snapshot1` recurring job with `default` in groups. Then should have 1 cron job. And volume `test-job-1` should have volume-head 1 snapshot. volume `test-job-2` should have volume-head 1 snapshot. When write some data to volume `test-job-1`. write some data to volume `test-job-2`. Then volume `test-job-1` should have 2 snapshots after scheduled time. volume `test-job-2` should have 2 snapshots after scheduled time. When write some data to volume `test-job-1`. write some data to volume `test-job-2`. And wait for `snapshot1` cron job scheduled time. Then volume `test-job-1` should have 3 snapshots after scheduled time. volume `test-job-2` should have 3 snapshots after scheduled time. #### Integration test - test_recurring_job_backup Scenario: test recurring job backup (S3/NFS) Given volume `test-job-1` created, attached, and healthy. volume `test-job-2` created, attached, and healthy. When create `backup1` recurring job with `default` in groups. Then should have 1 cron job. And volume `test-job-1` should have 0 backup. volume `test-job-2` should have 0 backup. When write some data to volume `test-job-1`. write some data to volume `test-job-2`. And wait for `backup1` cron job scheduled time. Then volume `test-job-1` should have 1 backups. volume `test-job-2` should have 1 backups. When write some data to volume `test-job-1`. write some data to volume `test-job-2`. And wait for `backup1` cron job scheduled time. Then volume `test-job-1` should have 2 backups. volume `test-job-2` should have 2 backups. #### Integration test - test_recurring_job_while_volume_detached Scenario: test recurring job while volume is detached Given volume `test-job-1` created, and detached. volume `test-job-2` created, and detached. And attach volume `test-job-1` and write some data. attach volume `test-job-2` and write some data. And detach volume `test-job-1`. detach volume `test-job-2`. When create `snapshot1` recurring job running at 1 minute interval, and with `default` in groups, and with `retain` set to `2`. And 1 cron job should be created. And wait for 2 minutes. Then attach volume `test-job-1` and wait until healthy. And volume `test-job-1` should have only 1 snapshot. When wait for 1 minute. Then volume `test-job-1` should have only 2 snapshots. When set setting `allow-recurring-job-while-volume-detached` to `true`. And wait for 2 minutes. Then attach volume `test-job-2` and wait until healthy. And volume `test-job-2` should have only 2 snapshots. #### Manual test - recurring job skip to create job while volume is detached Scenario: test recurring job while volume is detached Given volume `test-job-1` created, and detached. volume `test-job-2` created, and detached. When create `snapshot1` recurring job running at 1 minute interval, And wait until job pod created and complete Then monitor the job pod logs. And should see `Cannot create job for test-job-1 volume in state detached`. should see `Cannot create job for test-job-2 volume in state detached`. #### Manual test - recurring job upgrade migration Scenario: test recurring job upgrade migration Given cluster with Longhorn version prior to v1.2.0. And storageclass with recurring job `snapshot1`. And volume `test-job-1` created, and attached. When upgrade Longhorn to v1.2.0. Then should have recurring job CR created with format `---`. And volume should be labeled with `recurring-job.longhorn.io/---: enabled`. And recurringJob should be removed in volume spec. And storageClass in `longhorn-storageclass` configMap should not have `recurringJobs`. storageClass in `longhorn-storageclass` configMap should have `recurringJobSelector`. ``` recurringJobSelector: '[{"name":"snapshot-1-97893a05-77074ba4","isGroup":false},{"name":"backup-1-954b3c8c-59467025","isGroup":false}]' ``` When create new PVC. And volume should be labeled with items in `recurringJobSelector`. And recurringJob should not exist in volume spec. #### Manual test - snapshot concurrency Scenario: test recurring job concurrency Given create `snapshot1` recurring job with `concurrency` set to `2`. include `snapshot1` recurring job `default` in groups. When create volume `test-job-1`. create volume `test-job-2`. create volume `test-job-3`. create volume `test-job-4`. create volume `test-job-5`. Then monitor the cron job pod log. And should see 2 jobs created concurrently. When update `snapshot1` recurring job with `concurrency` set to `3`. Then monitor the cron job pod log. And should see 3 jobs created concurrently. ### Upgrade strategy #### Automated Migration 1. Create `v110to120/upgrade.go` 2. Translate `storageClass` `recurringJobs` to `recurringJobSelector`. 1. Convert the `recurringJobs` to `recurringJobSelector` object. ``` { Name: --- IsGroup: false, } ``` 2. Add `recurringJobSelector` to `longhorn-storageclass` configMap. 3. Remove `recurringJobs` in configMap. 4. Update configMap. ``` parameters: fromBackup: "" numberOfReplicas: "3" recurringJobSelector: '[{"name":"snapshot-1-97893a05-77074ba4","isGroup":false},{"name":"backup-1-954b3c8c-59467025","isGroup":false}]' staleReplicaTimeout: "2880" provisioner: driver.longhorn.io ``` 3. Translate volume spec `recurringJobs` to volume labels. 1. List all volumes and its spec `recurringJobs` and create labels in format `recurring-job.longhorn.io/---: enabled`. 2. Update volume labels and remove volume spec `recurringJobs`. ``` labels: longhornvolume: pvc-d37caaed-5cda-43b1-ae49-9d0490ffb3db recurring-job.longhorn.io/backup-1-954b3c8c-59467025: enabled recurring-job.longhorn.io/snapshot-1-97893a05-77074ba4: enabled ``` 3. translate volume spec `recurringJobs` to recurringJob CRs. 1. Gather the recurring jobs from `recurringJobSelector` and volume labels. 2. Create recurringJob CRs. ``` NAME GROUPS TASK CRON RETAIN CONCURRENCY AGE LABELS snapshot-1-97893a05-77074ba4 snapshot */1 * * * * 1 10 13m backup-1-954b3c8c-59467025 backup */2 * * * * 1 10 13m {"interval":"2m"} ``` 4. Cleanup applied volume cron jobs. 1. Get all applied cron jobs for volumes. 2. Delete cron jobs. > The migration translates existing volume recurring job with format `recurring-job.longhorn.io/---: enabled`. The name maps to the recurring job CR `---`. > The migration translates existing volume recurring job with format `recurring-job.longhorn.io/---: enabled`. The numbers could look random and also differs from the recurring job name of the CR name created by the StorageClass - `recurring-job.longhorn.io/: enabled`. This is because there is no info to determine if the volume spec `recurringJob` is coming from a `storageClass` or which `storageClass`. Should note this behavior in the document to lessen the confusion unless there is a better solution. #### Manual After the migration, the `-` in volume label and recurring job name could look random and confusing. Users might want to rename it to something more meaningful. Currently, the only way is to create a new recurring job CR and replace the volume label. ## Note [optional] `None` ================================================ FILE: enhancements/20210701-backing-image.md ================================================ # Backing Image v2 ## Summary Longhorn can set a backing image of a Longhorn volume, which is designed for VM usage. ### Related Issues https://github.com/longhorn/longhorn/issues/2006 https://github.com/longhorn/longhorn/issues/2295 https://github.com/longhorn/longhorn/issues/2530 https://github.com/longhorn/longhorn/issues/2404 ## Motivation ### Goals 1. A qcow2 or raw image file can be used as the backing image of a volume. 2. The backing image works fine with backup or restore. 3. Multiple replicas in the same disk can share one backing image. 4. The source of a backing image file can be remote downloading, upload, Longhorn volume, etc. 5. Once the first backing image file is ready, Longhorn can deliver it to other nodes. 6. Checksum verification for backing image. 7. HA backing image. ### Non-goals: This feature is not responsible for fixing issue mentioned in [issue #1948](https://github.com/longhorn/longhorn/issues/1948#issuecomment-791641308). ## Proposal 1. Each backing image is stored as a object of a new CRD named `BackingImage`. - The spec/status records if a disk requires/contains the related backing image file. - To meet backing image HA requirement, some ready disks are randomly picked. Besides, whether a disk requires the backing image file is determined by if there is replicas using it in the disk. - A file in a disk cannot be removed as long as there is a replica using it. 2. Longhorn needs to prepare the 1st backing image file based on the source type: - Typically, the 1st file preparation takes a short amount of time comparing with the whole lifecycle of BackingImage. Longhorn can use a temporary pod to handle it. Once the file is ready, Longhorn can stop the pod. - We can use a CRD named `BackingImageDataSource` to abstract the file preparation. The source type and the parameters will be in the spec. - To allow Longhorn manager query the progress of the file preparation, we should launch a service for this pod. Considering that multiple kinds of source like download or uploading involve HTTP, we can launch a HTTP server for the pod. 3. Then there should be a component responsible for monitoring and syncing backing image files with other nodes after the 1st file ready. - Similar to `BackingImageDataSource`, we will use a new CRD named `BackingImageManager` to abstract this component. - The BackingImageManager pod is design to - take over the ownership when the 1st file prepared by the BackingImageDataSource pod is ready. - deliver the file to others if necessary. - monitor all backing image files for a specific disk: Considering the disk migration and failed replica reusage features, there will be an actual backing image file for each disk rather than each node. - BackingImageManager should support reuse existing backing image files. Since we will consider those files are immutable/read-only once ready. If there is an expected checksum for a BackingImage, the pod will compare the checksum before reusing. - Live upgrade is possible: Different from instance managers, BackingImageManagers manage files only. We can directly shut down the old BackingImageManager pods, then let new BackingImageManager pods rely on the reuse mechanism to take over the existing files. - If the disk is not available/gets replaced, the BackingImageManager cannot do anything or simply report all BackingImages failed. - Once there is a modification for an image, managers will notify callers via the gRPC streaming. 4. `longhorn-manager` will launch & update controllers for these new CRDs: - BackingImageController is responsible for: 1. Generate a UUID for each new BackingImage. 2. Sync with the corresponding BackingImageDataSource: 3. Handle BackingImageManager life cycle. 4. Sync download status/info with BackingImageDataSource status or BackingImageManager status. 5. Set timestamp if there is no replica using the backing image file in a disk. - BackingImageDataSourceController is responsible for: 1. Sync with the corresponding BackingImage. 3. Handle BackingImageManager life cycle. 4. Sync download status/info with BackingImageDataSource status or BackingImageManager status. 5. Set timestamp if there is no replica using the BackingImage in a disk - BackingImageManagerController is responsible for: 1. Create pods to handle backing image files. 2. Handle files based on the spec & BackingImageDataSource status: - Delete unused BackingImages. - Fetch the 1st file based on BackingImageDataSource. Otherwise, sync the files from other managers or directly reuse the existing files. 5. For `longhorn-engine`: - Most of the backing image related logic is already in there. - The raw image support will be introduced. - Make sure the backing file path will be updated each time when the replica starts. - The lifecycle of the components: ``` |Created by HTTP API. |Set deletion timestamp, will delete BackingImageDataSource first. BackingImage: |========|============================================================================|======================================================| |Create BackingImageManagers |Deleted after cleanup. | base on HA or replica requirements. |Created by HTTP API after |Set deletion timestamp when | BackingImage creation. | BackingImage is being deleted. BackingImageDataSource: |===============|=========================================|=============|=================|=========================================| |Start a pod then |File ready. |Stop the pod when |Deleted. | file preparation immediately | BackingImageManager takes over the file. |Take over the 1st file from BackingImageDataSource. |Do cleanup if required |Created by BackingImageController | Or sync/receive files from peers. | then get deleted. BackingImageManager: |===========|==============|==================================|===============================================|============| |Start a pod. |Keep file monitoring |Set deletion timestamp since | after pod running. | no BackingImage is in the disk. ``` - BackingImage CRs and BackingImageDataSource CRs are one-to-one correspondence. One backingImageDataSource CR is always created after the related BackingImage CR, but deleted before the BackingImage CR cleanup. - The lifecycle of one BackingImageManager CR is not controlled by one single BackingImage. For a disk, the related BackingImageManager CR will be created as long as there is one BackingImage required. However, it will be removed only if there is no BackingImage in the disk. ### User Stories #### Rebuild replica for a large volume after network fluctuation/node reboot Before the enhancement, users need to manually copy the backing image data to the volume in advance. After the enhancement, users can directly specify the BackingImage during volume creation/restore with a click. And one backing image file can be shared among all replicas in the same disk. ### User Experience In Detail 1. Users can modify the backing image file cleanup timeout setting so that all non-used files will be cleaned up automatically from disks. 2. Create a volume with a backing image 2.1. via Longhorn UI 1. Users add a backing image, which is similar to add an engine image or set up the backup target in the system. 2. Users create/restore a volume with the backing image specified from the backing image list. 2.2. via CSI (StorageClass) - By specifying `backingImageName` in a StorageClass, all volumes created by this StorageClass will utilize this backing image. - If the optional fields `backingImageDataSourceType` and `backingImageDataSourceParameters` are set and valid, Longhorn will automatically create a volume as well as the backing image if the backing image does not exists. 3. Users attach the volume to a node (via GUI or Kubernetes). Longhorn will automatically prepare the related backing image to the disks the volume replica are using. In brief, users don't need to do anything more for the backing image. 4. When users backup a volume with a backing image, the backing image info will be recorded in the backup but the actual backing image data won't be uploaded to the backupstore. Instead, the backing image will be re-downloaded from the original image once it's required. ### API Changes - A bunch of RESTful APIs is required for the new CRD `BackingImage`: `Create`, `Delete`, `List`, and `BackingImageCleanup`. - Now the volume creation API receives parameter `BackingImage`. ## Design ### Implementation Overview #### longhorn-manager: 1. In settings: - Add a setting `Backing Image Cleanup Wait Interval`. - Add a read-only setting `Default Backing Image Manager Image`. 2. Add a new CRD `backingimages.longhorn.io`. ```goregexp type BackingImageSpec struct { Disks map[string]struct{} `json:"disks"` Checksum string `json:"checksum"` } type BackingImageStatus struct { OwnerID string `json:"ownerID"` UUID string `json:"uuid"` Size int64 `json:"size"` Checksum string `json:"checksum"` DiskFileStatusMap map[string]*BackingImageDiskFileStatus `json:"diskFileStatusMap"` DiskLastRefAtMap map[string]string `json:"diskLastRefAtMap"` } type BackingImageDiskFileStatus struct { State BackingImageState `json:"state"` Progress int `json:"progress"` Message string `json:"message"` } ``` ```goregexp const ( BackingImageStatePending = BackingImageState("pending") BackingImageStateStarting = BackingImageState("starting") BackingImageStateReady = BackingImageState("ready") BackingImageStateInProgress = BackingImageState("in_progress") BackingImageStateFailed = BackingImageState("failed") BackingImageStateUnknown = BackingImageState("unknown") ) ``` - Field `Spec.Disks` records the disks that requires this backing image. - Field `Status.DiskFileStatusMap` reflect the current file status for the disks. If there is anything wrong with the file, the error message can be recorded inside the status. - Field `Status.UUID` should be generated and stored in ETCD before other operations. Considering users may create a new BackingImage with the same name but different parameters after deleting an old one, to avoid the possible leftover of the old BackingImage disturbing the new one, the manager can use a UUID to generate the work directory. 3. Add a new CRD `backingimagedatasources.longhorn.io`. ```goregexp type BackingImageDataSourceSpec struct { NodeID string `json:"nodeID"` DiskUUID string `json:"diskUUID"` DiskPath string `json:"diskPath"` Checksum string `json:"checksum"` SourceType BackingImageDataSourceType `json:"sourceType"` Parameters map[string]string `json:"parameters"` Started bool `json:"started"` } type BackingImageDataSourceStatus struct { OwnerID string `json:"ownerID"` CurrentState BackingImageState `json:"currentState"` Size int64 `json:"size"` Progress int `json:"progress"` Checksum string `json:"checksum"` } type BackingImageDataSourceType string const ( BackingImageDataSourceTypeDownload = BackingImageDataSourceType("download") BackingImageDataSourceTypeUpload = BackingImageDataSourceType("upload") ) const ( DataSourceTypeDownloadParameterURL = "url" ) ``` - Field `Started` indicates if the BackingImageManager already takes over the file. Once this is set, Longhorn can stop the corresponding pod as well as updating the object itself. 4. Add a new CRD `backingimagemanagers.longhorn.io`. ```goregexp type BackingImageManagerSpec struct { Image string `json:"image"` NodeID string `json:"nodeID"` DiskUUID string `json:"diskUUID"` DiskPath string `json:"diskPath"` BackingImages map[string]string `json:"backingImages"` } type BackingImageManagerStatus struct { OwnerID string `json:"ownerID"` CurrentState BackingImageManagerState `json:"currentState"` BackingImageFileMap map[string]BackingImageFileInfo `json:"backingImageFileMap"` IP string `json:"ip"` APIMinVersion int `json:"apiMinVersion"` APIVersion int `json:"apiVersion"` } ``` ```goregexp type BackingImageFileInfo struct { Name string `json:"name"` UUID string `json:"uuid"` Size int64 `json:"size"` State BackingImageState `json:"state"` CurrentChecksum string `json:"currentChecksum"` Message string `json:"message"` SendingReference int `json:"sendingReference"` SenderManagerAddress string `json:"senderManagerAddress"` Progress int `json:"progress"` } ``` ```goregexp const ( BackingImageManagerStateError = BackingImageManagerState("error") BackingImageManagerStateRunning = BackingImageManagerState("running") BackingImageManagerStateStopped = BackingImageManagerState("stopped") BackingImageManagerStateStarting = BackingImageManagerState("starting") BackingImageManagerStateUnknown = BackingImageManagerState("unknown") ) ``` - Field `Spec.BackingImages` records which BackingImages should be monitored by the manager. the key is BackingImage name, the value is BackingImage UUID. - Field `Status.BackingImageFileMap` will be updated according to the actual file status reported by the related manager pod. - Struct `BackingImageFileInfo` is used to load the info from BackingImageManager pods. 5. Add a new controller `BackingImageDataSourceController`. 1. Important notices: 1. Once a BackingImageManager takes over the file ownership, the controller doesn't need to update the related BackingImageDataSource CR except for cleanup. 3. The state is designed to reflect the file state rather than the pod phase. Of course, the file state will be considered as failed if the pod somehow doesn't work correctly. e.g., the pod suddenly becomes failed or being removed. 2. Workflow: 1. Check and update the ownership. 2. Do cleanup if the deletion timestamp is set. Cleanup means stopping monitoring and kill the pod. 3. Sync with the BackingImage: 1. For in-progress BackingImageDataSource, Make sure the disk used by this BackingImageDataSource is recorded in the BackingImage spec as well. 2. [TODO] Guarantee the HA by adding more disks to the BackingImage spec once BackingImageDataSource is started. 4. Skip updating "started" BackingImageDataSource. 5. Handle pod: 1. Check the pod status. 2. Update the state based on the previous state and the current pod phase: 1. If the pod is ready for service, do nothing. 2. If the pod is not ready, but the file processing already start. It means there is something wrong with the flow. This BackingImageDataSource will be considered as `error`. 3. If the pod is failed, the BackingImageDataSource should be `error` as well. 4. When the pod reaches an unexpected phase or becomes failed, need to record the error message or error log in the pod. 3. Start or stop monitoring based on pod phase. 4. Delete the errored pod. 5. Create or recreate the pod, then update the backoff entry. Whether the pod can be recreated is determined by the backoff window and the source type. For the source types like upload, recreating pod doesn't make sense. Users need to directly do cleanup then recreate a new backing image instead. 6. For the monitor goroutine, it's similar to that in InstanceManagerController. - It will `Get` the file info via HTTP every 3 seconds. - If there are 10 continuous HTTP failures, the monitor goroutine will stop itself. Then the controller will restart it. - If the backing image is ready, clean up the entry in the backoff. 6. Add a new controller `BackingImageManagerController`. 1. Important notices: 1. Need to consider 2 kinds of managers: default manager, old manager(this includes all incompatible managers). 2. All old managers will be removed immediately once there is the default image is updated. And old managers shouldn't operate any backing image files. - When an old manager is removed, the files inside in won't be gone. These files will be taken by the new one. By disabling old managers operating the files, the conflicts with the default manager won't happen. - The controller can directly delete old BackingImageManagers without affecting existing BackingImages. This simplifies the cleanup flow. - Ideally there should be a cleanup mechanism that is responsible for removing all failed backing image files as well as the images no longer required by the new BackingImageManagers. But due to lacking of time, it will be implemented in the future. 3. In most cases, the controller and the BackingImageManager will avoid deleting backing images files.: - For example, if the pod is crashed or one image file becomes failed, the controller will directly restart the pod or re-download the image, rather than cleaning up the files only. - The controller will delete image files for only 2 cases: A BackingImage is no longer valid; A default BackingImageManager is deleted. - By following this strategy, we may risk at leaving some unused backing image files in some corner cases. However, the gain is that, there is lower probability of crashing a replica caused by the backing image file deletion. Besides, the existing files can be reused after recovery. And after introducing the cleanup mechanism, we should worry about the leftover anymore. 4. With passive file cleanup strategy, default managers can directly pick up all existing files via `Fetch` requests when the old manager pods are killed. This is the essential of live upgrade. 5. The pod not running doesn't mean all files handled by the pod become invalid. All files can be reused/re-monitored after the pod restarting. 2. Workflow: 1. If the deletion timestamp is set, the controller will clean up files for running default BackingImageManagers only. Then it will blindly delete the related pods. 2. When the disk is not ready, the current manager will be marked as `unknown`. Then all not-failed file records are considered as `unknown` as well. - Actually there are multiple subcases here: node down, node reboot, node disconnection, disk detachment, longhorn manager pod missing etc. It's complicated to distinguish all subcases to do something special. Hence, I choose simply marking the state to `unknown`. 3. Create BackingImageManager pods for. - If the old status is `running` but the pod is not ready now, there must be something wrong with the manager pod. Hence the controller need to update the state to `error`. - When the pod is ready, considering the case that the pod creation may succeed but the CR status update will fail due to conflicts, the controller won't check the previous state. Instead, it will directly update state to `running`. - Start a monitor goroutine for each running pods. - If the manager is state `error`, the controller will do cleanup then recreate the pod. 4. Handle files based on the spec: - Delete invalid files: - The BackingImages is no longer in `BackingImageManager.Spec.BackingImages`. - The BackingImage UUID doesn't match. - Make files ready for the disk: 1. When BackingImageDataSource is not "started", it means BackingImageManager hasn't taken over the 1st file. Once BackingImageDataSource reports file ready, BackingImageManager can get the 1st file via API `Fetch`. 2. Then if BackingImageDataSource is "started" but there is no ready record for a BackingImage among all managers, it means the pod someshow restarted (may due to upgrade). In this case, BackingImageManager can try to reuse the files via API `Fetch` as well. 3. Otherwise, the current manager will try to sync the file with other managers: - If the 1st file is not ready, do nothing. - Each manager can send a ready file to 3 other managers simultaneously at max. When there is no available sender, do nothing. 4. Before reusing or syncing files, the controller need to check the backoff entry for the corresponding BackingImageManager. And after the API call, the backoff entry will be updated. 5. For the monitor goroutine, it's similar to that in InstanceManagerController. - It will `List` all backing image files once it receives the notification from the streaming. - If there are 10 continuous errors returned by the streaming receive function, the monitor goroutine will stop itself. Then the controller will restart it. - Besides, if a backing image is ready, the monitor should clean up the entry from the backoff of the BackingImageManager. 7. Add a new controller `BackingImageController`. 1. Important notices: 1. One main responsibility of this controller is creating, deleting, and update BackingImageManagers. It is not responsible for communicating with BackingImageManager pods or BackingImageDataSource pods. 2. This controller can reset "started" BackingImageDataSource if all its backing image files are errored in the cluster and the source type is satisfied. 3. The immutable UUID should be generated and stored in ETCD before any other update. This UUID can can be used to distinguish a new BackingImage from an old BackingImage using the same name. 4. Beside recording the immutable UUID, the BackingImage status is used to record the file info in the managers status and present to users. 5. Always try to create default BackingImageManagers if not exist. 6. Aggressively delete non-default BackingImageManagers. 2. Workflow: 1. If the deletion timestamp is set, the controller need to do cleanup for all related BackingImageManagers as well as BackingImageDataSource. 2. Generate a UUID for each new BackingImage. Make sure the UUID is stored in ETCD before doing anything others. 3. Init fields in the BackingImage status. 4. Sync with BackingImageDataSource: 1. Mark BackingImageDataSource as started if the default BackingImageManager already takes over the file ownership. 2. When all files failed, mark the BackingImageDataSource when the source type is downloaded. Then it can re-download the file and recover this BackingImage. 3. Guarantee the disk info in BackingImageDataSources spec is correct if it's not started. (This can be done in Node Controller as well.) 5. Handle BackingImageManager life cycle: - Remove records in `Spec.BackingImages` or directly delete the manager CR - Add records to `Spec.BackingImages` for the current BackingImage. Create BackingImageManagers with default image if not exist. 6. Sync download status/info with BackingImageManager status: - If BackingImageDataSource is not started, update BackingImage status based on BackingImageDataSource status. Otherwise, sync status with BackingImageManagers. - Set `Status.Size` if it's 0. If somehow the size is not same among all BackingImageManagers, this means there is an unknown bug. Similar logic applied to `Status.CurrentChecksum`. 7. Set timestamp in `Status.DiskLastRefAtMap` if there is no replica using the BackingImage in a disk. Later NodeController will do cleanup for `Spec.DiskDownloadMap` based on the timestamp. Notice that this clean up should not break the backing image HA. 1. Try to set timestamps for disks in which there is no replica/BackingImageDataSource using this BackingImage first. 2. If there is no enough ready files after marking, remove timestamps for some disks that contain ready files. 3. If HA requirement is not satisfied when all ready files are retained, remove timestamps for some disks that contain in-progress/pending files. 4. If HA requirement is not unsatisfied, remove timestamps for some disks that contain failed files. Later Longhorn can try to do recovery for the disks contains these failed files. 6. In Replica Controller: - Request preparing the backing image file in a disk if a BackingImage used by a replica doesn't exist. - Check and wait for BackingImage disk map in the status before sending requests to replica instance managers. 7. In Node Controller: - Determine if the disk needs to be cleaned up if checking BackingImage `Status.DiskLastRefAtMap` and the wait interval `BackingImageCleanupWaitInterval`. - Update the spec for BackingImageManagers when there is a disk migration. 8. For the HTTP APIs - Volume creation: - Longhorn needs to verify the BackingImage if it's specified. - For restore/DR volumes, the BackingImage name stored in the backup volume will be used automatically if users do not specify the BackingImage name. Verify the checksum before using the BackingImage. - Snapshot backup: - BackingImage name and checksum will be record into BackupVolume now. - BackingImage creation: - Need to create both BackingImage CR and the BackingImageDataSource CR. Besides, a random ready disk will be picked up so that Longhorn can prepare the 1st file for the BackingImage immediately. - BackingImage get/list: - Be careful about the BackingImageDataSource not found error. There are 2 cases that would lead to this error: - BackingImageDataSource has not been created. Add retry would solve this case. - BackingImageDataSource is gone but BackingImage has not been cleaned up. Longhorn can ignore BackingImageDataSource when BackingImage deletion timestamp is set. - BackingImage disk cleanup: - This cannot break the HA besides attaching replicas. The main idea is similar to the cleanup in BackingImage Controller. 9. In CSI: - Check the backing image during the volume creation. - The missing BackingImage will be created when both BackingImage name and data source info are provided. #### longhorn-engine: - Verify the existing implementation and the related integration tests. - Add raw backing file support. - Update the backing file info for replicas when a replica is created/opened. #### backing-image-manager: ##### data source service: - A HTTP server will be launched to prepare the 1st BackingImage file based on the source type. - The server will download the file immediately once the type is `download` and the server is up. - A cancelled context will be put the HTTP download request. When the server is stopped/failed while downloading is still in-progress, the context can help stop the download. - The service will wait for 30s at max for download start. If time exceeds, the download is considered as failed. - The download file is in `/tmp/-` - Each time when the image downloads a chunk of data, the progress will be updated. For the first time updating the progress, it means the downloading starts and the state will be updated from `starting` to `in-progress`. - The server is ready for handling the uploaded data once the type is `upload` and the server is up. - The query `size` is required for the API `upload`. - The API `upload` receives a multi-part form request. And the body request is the file data streaming. - Similar to the download, the progress will be updated as long as the API receives and stores a chunk of data. For the first time updating the progress, it means the uploading starts and the state will be updated from `starting` to `in-progress`. ##### manager service: - A gRPC service will be launched to monitor and sync BackingImages: - API `Fetch`: Register the image then move the file prepared by BackingImageDataSource server to the image work directory. The file is typically in a tmp directory - If the file name is not specified in the request, it means reusing the existing file only. - For a failed BackingImage, the manager will re-register then re-fetch it. - Before fetching the file, the BackingImage will check if there are existing files in the current work directory. It the files exist and the checksum matches, the file will be directly reused and the config file is updated. - Otherwise, the work directory will be cleaned up and recreated. Then the file in the tmp directory will be moved to the work directory. - API `Sync`: Register the image, start a receiving server, and ask another manager to send the file via API `Send`. For a failed BackingImage, the manager will re-register then re-sync it. This should be similar to replica rebuilding. - Similar to `Fetch`, the image will try to reuse existing files. - The manager is responsible for managing all port. The image will use the functions provided by the manager to get then release ports. - API `Send`: Send a backing image file to a receiver. This should be similar to replica rebuilding. - API `Delete`: Unregister the image then delete the image work directory. Make sure syncing or pulling will be cancelled if exists. - API `Get`/`List`: Collect the status of one backing image file/all backing image files. - API `Watch`: establish a streaming connection to report BackingImage file info. - As I mentioned above, we will use BackingImage UUID to generate work directories for each BackingImage. The work directory is like: ``` /backing-images/ /backing-images/-/backing.tmp /backing-images/-/backing /backing-images/-/backing.cfg ``` - There is a goroutine periodically check the file existence based on the image file current state. - It will verify the disk UUID in the disk config file. If there is a mismatching, it will stop checking existing files. And the calls, longhorn manager pods, won't send requests since this BackingImageManager is marked as `unknown`. - The manager will provide one channel for all BackingImages. If there is an update in a BackingImage, the image will send a signal to the channel. Then there is another goroutine receive the channel and notify the longhorn manager via the streaming created by API `Watch`. #### longhorn-ui: 1. Launch a new page to present and operate BackingImages. 1. Add button `Create Backing Image` on the top right of the page: - Field `name` is required and should be unique. - Field `sourceType` is required and accept an enum value. This indicates how Longhorn can get the backing image file. Right now there are 2 options: `download`, `upload`. In the future, it can be value `longhorn-volume`. - Field `parameters` is a string map and is determined by `sourceType`. If the source type is `download`, the map should contain key `url`, whose value is the actual download address. If the source type is `upload`, the map is empty. - Field `expectedChecksum` is optional. The user can specify the SHA512 checksum of the backing image. When the backing image fetched by Longhorn doesn't match the non-empty expected value, the backing image won't be `ready`. 2. If the source type of the creation API is `upload`, UI should send a `upload` request with the actual file data when the upload server is ready for receiving. The upload server ready is represented by first disk file state becoming `starting`. UI can check the state and wait for up to 30 seconds before sending the request. 3. Support batch deletion: Allow selecting multiple BackingImages; Add button `Deletion` on the top left. 4. The columns on BackingImage list page should be: Name, Size, Created From (field `sourceType`), Operation. 5. Show more info for each BackingImage after clicking the name: - Present `Created From` (field `sourceType`) and the corresponding parameters `Parameters During Creation` (field `parameters`). - If `sourceType` is `download`, present `DOWNLOAD FROM URL` instead. - Show fields `expectedChecksum` and `currentChecksum` as `Expected SHA512 Checksum` and `Current SHA512 Checksum`. If `expectedChecksum` is empty, there is no need to show `Expected SHA512 Checksum`. - Use a table to present the file status for each disk based on fields `diskFileStatusMap`: - `diskFileStatusMap[diskUUID].progress` will be shown only when the state is `in-progress`. - Add a tooltip to present `diskFileStatusMap[diskUUID].message` if it's not empty. 6. Add the following operations under button `Operation`: - `Delete`: No field is required. It should be disabled when there is one replica using the BackingImage. - `Clean Up`: A disk file table will be presented. Users can choose the entries of this table as the input `disks` of API `CleanupDiskImages`. This API is dedicated for manually cleaning up the images in some disks in advance. 7. When a BackingImage is being deleted (field `deletionTimestamp` is not empty), show an icon behind the name which indicates the deletion state. 8. If the state of all disk records are `failed`, use an icon behind the name to indicates the BackingImage unavailable. 2. Allow choosing a BackingImage for volume creation. 3. Modify Backup page for BackingImage: - Allow choosing/re-specifying a new BackingImage for restore/DR volume creation: - If there is BackingImage info in the backup volume, an option `Use previous backing image` will be shown and checked by default. - If the option is unchecked by users, UI will show the BackingImage list so that users can pick up it. - Add a button `Backing Image Info` in the operation list: - If the backing image name of a BackupVolume is empty, gray out the button. - Otherwise, present the backing image name and the backing image checksum. | HTTP Endpoint | Operation | | -------------------------------------------------------------- | ------------------------------------------------------------------------ | | **GET** `/v1/backingimages` | Click button `Backing Image` | | **POST** `/v1/backingimages/` | Click button `Create Backing Image` | | **DELETE** `/v1/backingimages/{name}` | Click button `Delete` | | **GET** `/v1/backingimages/{name}` | Click the `name` of a backing image | | **POST** `/v1/backingimages/{name}?action=backingImageCleanup` | Click button`Clean Up` | | **POST** `/v1/backingimages/{name}?action=upload` | Longhorn UI should call it automatically when the upload server is ready | ### Test Plan #### Integration tests 1. Backing image basic operation 2. Backing image auto cleanup 3. Backing image with disk migration #### Manual tests 1. The backing image on a down node 2. The backing image works fine with system upgrade & backing image manager upgrade 3. The incompatible backing image manager handling 4. The error presentation of a failed backing image ### Upgrade strategy N/A ================================================ FILE: enhancements/20210810-volume-clone.md ================================================ # Support CSI Volume Cloning ## Summary We want to support CSI volume cloning so users can create a new PVC that has identical data as a source PVC. ### Related Issues https://github.com/longhorn/longhorn/issues/1815 ## Motivation ### Goals * Support exporting the snapshot data of a volume * Allow user to create a PVC with identical data as the source PVC ## Proposal There are multiple parts in implementing this feature: ### Sparse-tools Implement a function that fetches data from a readable object then sends it to a remote server via HTTP. ### Longhorn engine * Implementing `VolumeExport()` gRPC in replica SyncAgentServer. When called, `VolumeExport()` exports volume data at the input snapshot to the receiver on the remote host. * Implementing `SnapshotCloneCmd()` and `SnapshotCloneStatusCmd()` CLIs. Longhorn manager can trigger the volume cloning process by calling `SnapshotCloneCmd()` on the replica of new volume. Longhorn manager can fetch the cloning status by calling `SnapshotCloneStatusCmd()` on the replica of the new volume. ### Longhorn manager * When the volume controller detects that a volume clone is needed, it will attach the target volume. Start 1 replica for the target volume. Auto-attach the source volume if needed. Take a snapshot of the source volume. Copy the snapshot from a replica of the source volume to the new replica by calling `SnapshotCloneCmd()`. After the snapshot was copied over to the replica of the new volume, the volume controller marks volume as completed cloning. * Once the cloning is done, the volume controller detaches the source volume if it was auto attached. Detach the target volume to allow the workload to start using it. Later on, when the target volume is attached by workload pod, Longhorn will start rebuilding other replicas. ### Longhorn CSI plugin * Advertise that Longhorn CSI driver has ability to clone a volume, `csi.ControllerServiceCapability_RPC_CLONE_VOLUME` * When receiving a volume creat request, inspect `req.GetVolumeContentSource()` to see if it is from another volume. If so, create a new Longhorn volume with appropriate `DataSource` set so Longhorn volume controller can start cloning later on. ### User Stories Before this feature, to create a new PVC with the same data as another PVC, the users would have to use one of the following methods: 1. Create a backup of the source volume. Restore the backup to a new volume. Create PV/PVC for the new volume. This method requires a backup target. Data has to move through an extra layer (the backup target) which might cost money. 1. Create a new PVC (that leads to creating a new Longhorn volume). Mount both new PVC and source PVC to the same pod then copy the data over. See more [here](https://github.com/longhorn/longhorn/blob/v1.1.2/examples/data_migration.yaml). This copying method only applied for PVC with `Filesystem` volumeMode. Also, it requires manual steps. After this cloning feature, users can clone a volume by specifying `dataSource` in the new PVC pointing to an existing PVC. ### User Experience In Detail Users can create a new PVC that uses `longhorn` storageclass from an existing PVC which also uses `longhorn` storageclass by specifying `dataSource` in new PVC pointing to the existing PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: clone-pvc namespace: myns spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 5Gi dataSource: kind: PersistentVolumeClaim name: source-pvc ``` ### API changes `VolumeCreate` API will check/validate data a new field, `DataSource` which is a new field in `v.Spec` that specifies the source of the Longhorn volume. ## Design ### Implementation Overview ### Sparse-tools Implement a generalized function, `SyncContent()`, which syncs the content of a `ReaderWriterAt` object to a file on remote host. The `ReaderWriterAt` is interface that has `ReadAt()`, `WriteAt()` and `GetDataLayout()` method: ```go type ReaderWriterAt interface { io.ReaderAt io.WriterAt GetDataLayout (ctx context.Context) (<-chan FileInterval, <-chan error, error) } ``` Using those methods, the Sparse-tools know where is a data/hole interval to transfer to a file on the remote host. ### Longhorn engine * Implementing `VolumeExport()` gRPC in replica SyncAgentServer. When called, `VolumeExport()` will: * Create and open a read-only replica from the input snapshot * Pre-load `r.volume.location` (which is the map of data sector to snapshot file) by: * If the volume has backing file layer and users want to export the backing file layer, we initialize all elements of `r.volume.location` to 1 (the index of the backing file layer). Otherwise, initialize all elements of `r.volume.location` to 0 (0 means we don't know the location for this sector yet) * Looping over `r.volume.files` and populates `r.volume.location` (which is the map of data sector to snapshot file) with correct values. * The replica is able to know which region is data/hole region. This logic is implemented inside the replica's method `GetDataLayout()`. The method checks `r.volume.location`. The sector at offset `i` is in data region if `r.volume.location[i] >= 1`. Otherwise, the sector is inside a hole region. * Call and pass the read-only replica into `SyncContent()` function in the Sparse-tools module to copy the snapshot to a file on the remote host. * Implementing `SnapshotCloneCmd()` and `SnapshotCloneStatusCmd()` CLIs. * Longhorn manager can trigger the volume cloning process by calling `SnapshotCloneCmd()` on the replica of the new volume. The command finds a healthy replica of the source volume by listing replicas of the source controller and selecting a `RW` replica. The command then calls `CloneSnapshot()` method on replicas of the target volumes. This method in turn does: * Call `SnapshotClone()` on the sync agent of the target replica. This will launch a receiver server on the target replica. Call `VolumeExport()` on the sync agent of the source replica to export the snapshot data to the target replica. Once the snapshot data is copied over, revert the target replica to the newly copied snapshot. * Longhorn manager can fetch cloning status by calling `SnapshotCloneStatusCmd()` on the target replica. ### Longhorn manager * Add a new field to volume spec, `DataSource`. The `DataSource` is of type `VolumeDataSource`. Currently, there are 2 types of data sources: `volume` type and `snapshot` type. `volume` data source type has the format `vol://`. `snapshot` data source type has the format `snap:///`. In the future, we might want to refactor `fromBackup` field into a new type of data source with format `bk:///`. * Add a new field into volume status, `CloneStatus` of type `VolumeCloneStatus`: ```go type VolumeCloneStatus struct { SourceVolume string `json:"sourceVolume"` Snapshot string `json:"snapshot"` State VolumeCloneState `json:"state"` } type VolumeCloneState string const ( VolumeCloneStateEmpty = VolumeCloneState("") VolumeCloneStateInitiated = VolumeCloneState("initiated") VolumeCloneStateCompleted = VolumeCloneState("completed") VolumeCloneStateFailed = VolumeCloneState("failed") ) ``` * Add a new field into engine spec, `RequestedDataSource` of type `VolumeDataSource` * Add a new field into engine status, `CloneStatus`. `CloneStatus` is a map of `SnapshotCloneStatus` inside each replica: ```go type SnapshotCloneStatus struct { IsCloning bool `json:"isCloning"` Error string `json:"error"` Progress int `json:"progress"` State string `json:"state"` FromReplicaAddress string `json:"fromReplicaAddress"` SnapshotName string `json:"snapshotName"` } ``` This will keep track of status of snapshot cloning inside the target replica. * When the volume controller detect that a volume clone is needed (`v.Spec.DataSource` is `volume` or `snapshot` type and `v.Status.CloneStatus.State == VolumeCloneStateEmpty`), it will auto attach the source volume if needed. Take a snapshot of the source volume if needed. Fill the `v.Status.CloneStatus` with correct value for `SourceVolume`, `Snapshot`, and `State`(`initiated`). Auto attach the target volume. Start 1 replica for the target volume. Set `e.Spec.RequestedDataSource` to the correct value, `snap://`. * Engine controller monitoring loop will start the snapshot clone by calling `SnapshotCloneCmd()`. * After the snapshot is copied over to the replica of the new volume, volume controller marks `v.Status.CloneStatus.State = VolumeCloneStateCompleted` and clear the `e.Spec.RequestedDataSource` * Once the cloning is done, the volume controller detaches the source volume if it was auto attached. Detach the target volume to allow the workload to start using it. * When workload attach volume, Longhorn starts rebuilding other replicas of the volume. ### Longhorn CSI plugin * Advertise that Longhorn CSI driver has ability to clone a volume, `csi.ControllerServiceCapability_RPC_CLONE_VOLUME` * When receiving a volume creat request, inspect `req.GetVolumeContentSource()` to see if it is from another volume. If so, create a new Longhorn volume with appropriate `DataSource` set so Longhorn volume controller can start cloning later on. ### Test plan Integration test plan. #### Clone volume that doesn't have backing image 1. Create a PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: source-pvc spec: storageClassName: longhorn accessModes: - ReadWriteOnce resources: requests: storage: 10Gi ``` 1. Specify the `source-pvc` in a pod yaml and start the pod 1. Wait for the pod to be running, write some data to the mount path of the volume 1. Clone a volume by creating the PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cloned-pvc spec: storageClassName: longhorn dataSource: name: source-pvc kind: PersistentVolumeClaim accessModes: - ReadWriteOnce resources: requests: storage: 10Gi ``` 1. Specify the `cloned-pvc` in a cloned pod yaml and deploy the cloned pod 1. Wait for the `CloneStatus.State` in `cloned-pvc` to be `completed` 1. In 3-min retry loop, wait for the cloned pod to be running 1. Verify the data in `cloned-pvc` is the same as in `source-pvc` 1. In 2-min retry loop, verify the volume of the `clone-pvc` eventually becomes healthy 1. Cleanup the cloned pod, `cloned-pvc`. Wait for the cleaning to finish 1. Scale down the source pod so the `source-pvc` is detached. 1. Wait for the `source-pvc` to be in detached state 1. Clone a volume by creating the PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cloned-pvc spec: storageClassName: longhorn dataSource: name: source-pvc kind: PersistentVolumeClaim accessModes: - ReadWriteOnce resources: requests: storage: 10Gi ``` 1. Specify the `cloned-pvc` in a cloned pod yaml and deploy the cloned pod 1. Wait for `source-pvc` to be attached 1. Wait for a new snapshot created in `source-pvc` volume created 1. Wait for the `CloneStatus.State` in `cloned-pvc` to be `completed` 1. Wait for `source-pvc` to be detached 1. In 3-min retry loop, wait for the cloned pod to be running 1. Verify the data in `cloned-pvc` is the same as in `source-pvc` 1. In 2-min retry loop, verify the volume of the `clone-pvc` eventually becomes healthy 1. Cleanup the test #### Clone volume that has backing image 1. Deploy a storage class that has backing image parameter ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-bi-parrot provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes backingImage: "bi-parrot" backingImageURL: "https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.qcow2" ``` Repeat the `Clone volume that doesn't have backing image` test with `source-pvc` and `cloned-pvc` use `longhorn-bi-parrot` instead of `longhorn` storageclass #### Interrupt volume clone process 1. Create a PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: source-pvc spec: storageClassName: longhorn accessModes: - ReadWriteOnce resources: requests: storage: 10Gi ``` 1. Specify the `source-pvc` in a pod yaml and start the pod 1. Wait for the pod to be running, write 1GB of data to the mount path of the volume 1. Clone a volume by creating the PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cloned-pvc spec: storageClassName: longhorn dataSource: name: source-pvc kind: PersistentVolumeClaim accessModes: - ReadWriteOnce resources: requests: storage: 10Gi ``` 1. Specify the `cloned-pvc` in a cloned pod yaml and deploy the cloned pod 1. Wait for the `CloneStatus.State` in `cloned-pvc` to be `initiated` 1. Kill all replicas process of the `source-pvc` 1. Wait for the `CloneStatus.State` in `cloned-pvc` to be `failed` 1. In 2-min retry loop, verify cloned pod cannot start 1. Clean up cloned pod and `clone-pvc` 1. Redeploy `cloned-pvc` and clone pod 1. In 3-min retry loop, verify cloned pod become running 2. `cloned-pvc` has the same data as `source-pvc` 1. Cleanup the test ### Upgrade strategy No upgrade strategy needed ================================================ FILE: enhancements/20220110-extend-csi-snapshot-to-support-longhorn-snapshot.md ================================================ # Title Extend CSI snapshot to support Longhorn snapshot ## Summary Before this feature, if the user uses [the CSI Snapshotter mechanism](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html), they can only create Longhorn backups (out of cluster). We want to extend the CSI Snapshotter to support creating for Longhorn snapshot (in-cluster) as well. ### Related Issues https://github.com/longhorn/longhorn/issues/2534 ## Motivation ### Goals Extend the CSI Snapshotter to support: * Creating Longhorn snapshot * Deleting Longhorn snapshot * Creating a new PVC from a CSI snapshot that is associated with a Longhorn snapshot ### Non-goals * Longhorn snapshot Reverting is not a goal because CSI snapshotter doesn't support replace in place for now: https://github.com/container-storage-interface/spec/blob/master/spec.md#createsnapshot ## Proposal ### User Stories Before this feature is implemented, users can only use CSI Snapshotter to create/restore Longhorn backups. This means that users must set up a backup target outside of the cluster. Uploading/downloading data from backup target is a long/costly operation. Sometimes, users might just want to use CSI Snapshotter to take an in-cluster Longhorn snapshot and create a new volume from that snapshot. The Longhorn snapshot operation is cheap and faster than the backup operation and doesn't require setting up a backup target. ### User Experience In Detail To use this feature, users need to do: 1. Deploy the CSI snapshot CRDs, Controller as instructed at https://longhorn.io/docs/1.2.3/snapshots-and-backups/csi-snapshot-support/enable-csi-snapshot-support/ 1. Deploy a VolumeSnapshotClass with the parameter `type: longhorn-snapshot`. I.e., ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: longhorn-snapshot driver: driver.longhorn.io deletionPolicy: Delete parameters: type: longhorn-snapshot ``` 1. To create a new CSI snapshot associated with a Longhorn snapshot of the volume `test-vol`, users deploy the following VolumeSnapshot CR: ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot spec: volumeSnapshotClassName: longhorn-snapshot source: persistentVolumeClaimName: test-vol ``` A new Longhorn snapshot is created for the volume `test-vol` 1. To create a new PVC from the CSI snapshot, users can deploy the following yaml: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-snapshot-pvc spec: storageClassName: longhorn dataSource: name: test-snapshot kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi # should be the same as the size of `test-vol` ``` A new PVC will be created with the same content as in the VolumeSnapshot `test-snapshot` 1. Deleting the VolumeSnapshot `test-snapshot` will lead to the deletion of the corresponding Longhorn snapshot of the volume `test-vol` ### API changes None ## Design ### Implementation Overview We follow the specification in [the CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md#createsnapshot) when supporting the CSI snapshot. We define a new parameter in the VolumeSnapshotClass `type`. The value of the parameter `type` can be `longhorn-snapshot` or `longhorn-backup`. When `type` is `longhorn-snapshot` it means that the CSI VolumeSnapshot created with this VolumeSnapshotClass is associated with a Longhorn snapshot. When `type` is `longhorn-backup` it means that the CSI VolumeSnapshot created with this VolumeSnapshotClass is associated with a Longhorn backup. In [CreateSnapshot function](https://github.com/longhorn/longhorn-manager/blob/878cfb868c568396d6ebfa4ce096c5d95d9b31e3/csi/controller_server.go#L539), we get the value of parameter `type`. If it is `longhorn-backup`, we take a Longhorn backup as before. If it is `longhorn-snapshot` we do: * Get the name of the Longhorn volume * Check if the volume is in attached state. If it is not, return `codes.FailedPrecondition`. We cannot take a snapshot of non-attached volume. * Check if a Longhorn snapshot with the same name as the requested CSI snapshot already exists. If yes, return OK without taking a new Longhorn snapshot. * Take a new Longhorn snapshot. Encode the snapshotId in the format `snap://volume-name/snapshot-name`. This snapshotId will be used in the later CSI CreateVolume and DeleteSnapshot call. In [CreateVolume function](https://github.com/longhorn/longhorn-manager/blob/878cfb868c568396d6ebfa4ce096c5d95d9b31e3/csi/controller_server.go#L63): * If the VolumeContentSource is a `VolumeContentSource_Snapshot` type, decode the snapshotId in the format from the above step. * Create a new volume with the `dataSource` set to `snap://volume-name/snapshot-name`. This will trigger Longhorn to clone the content of the snapshot to the new volume. Note that if the source volume is not attached, Longhorn cannot verify the existence of the snapshot inside the Longhorn volume. This means that [the API will return error](https://github.com/longhorn/longhorn-manager/blob/878cfb868c568396d6ebfa4ce096c5d95d9b31e3/manager/volume.go#L347-L352) and new PVC cannot be provisioned. In [DeleteSnapshot function](https://github.com/longhorn/longhorn-manager/blob/878cfb868c568396d6ebfa4ce096c5d95d9b31e3/csi/controller_server.go#L675): * Decode the snapshotId in the format from the above step. If the type is `longhorn-backup` we delete the backup as before. If the type is `longhorn-snapshot`, we delete the corresponding Longhorn snapshot of the source volume. If the source volume or the snapshot is no longer exist, we return OK as specified in [the CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md#deletesnapshot) ### Test plan Integration test plan. 1. Deploy the CSI snapshot CRDs, Controller as instructed at https://longhorn.io/docs/1.2.3/snapshots-and-backups/csi-snapshot-support/enable-csi-snapshot-support/ 1. Deploy 4 VolumeSnapshotClass: ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: longhorn-backup-1 driver: driver.longhorn.io deletionPolicy: Delete ``` ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: longhorn-backup-2 driver: driver.longhorn.io deletionPolicy: Delete parameters: type: longhorn-backup ``` ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: longhorn-snapshot driver: driver.longhorn.io deletionPolicy: Delete parameters: type: longhorn-snapshot ``` ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1beta1 metadata: name: invalid-class driver: driver.longhorn.io deletionPolicy: Delete parameters: type: invalid ``` 1. Create Longhorn volume `test-vol` of 5GB. Create PV/PVC for the Longhorn volume. 1. Create a workload that uses the volume. Write some data to the volume. Make sure data persist to the volume by running `sync` 1. Set up a backup target for Longhorn #### Scenarios 1: CreateSnapshot * `type` is `longhorn-backup` or `""` * Create a VolumeSnapshot with the following yaml ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-longhorn-backup spec: volumeSnapshotClassName: longhorn-backup-1 source: persistentVolumeClaimName: test-vol ``` * Verify that a backup is created. * Delete the `test-snapshot-longhorn-backup` * Verify that the backup is deleted * Create the `test-snapshot-longhorn-backup` VolumeSnapshot with `volumeSnapshotClassName: longhorn-backup-2` * Verify that a backup is created. * `type` is `longhorn-snapshot` * volume is in detached state. * Scale down the workload of `test-vol` to detach the volume. * Create `test-snapshot-longhorn-snapshot` VolumeSnapshot with `volumeSnapshotClassName: longhorn-snapshot`. * Verify the error `volume ... invalid state ... for taking snapshot` in the Longhorn CSI plugin. * volume is in attached state. * Scale up the workload to attach `test-vol` * Verify that a Longhorn snapshot is created for the `test-vol`. * invalid type * Create `test-snapshot-invalid` VolumeSnapshot with `volumeSnapshotClassName: invalid-class`. * Verify the error `invalid snapshot type: %v. Must be %v or %v or` in the Longhorn CSI plugin. * Delete `test-snapshot-invalid` VolumeSnapshot. #### Scenarios 2: Create new volume from CSI snapshot * From `longhorn-backup` type * Create a new PVC with the flowing yaml: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-pvc spec: storageClassName: longhorn dataSource: name: test-snapshot-longhorn-backup kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` * Attach the PVC `test-restore-pvc` and verify the data * Delete the PVC * From `longhorn-snapshot` type * Source volume is attached && Longhorn snapshot exist * Create a PVC with the following yaml: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-pvc spec: storageClassName: longhorn dataSource: name: test-snapshot-longhorn-snapshot kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` * Attach the PVC `test-restore-pvc` and verify the data * Delete the PVC * Source volume is detached * Scale down the workload to detach the `test-vol` * Create the same PVC `test-restore-pvc` as in the `Source volume is attached && Longhorn snapshot exist` section * Verify that PVC provisioning failed because the source volume is detached so Longhorn cannot verify the existence of the Longhorn snapshot in the source volume. * Scale up the workload to attach `test-vol` * Wait for PVC to finish provisioning and be bounded * Attach the PVC `test-restore-pvc` and verify the data * Delete the PVC * Source volume is attached && Longhorn snapshot doesn’t exist * Find the VolumeSnapshotContent of the VolumeSnapshot `test-snapshot-longhorn-snapshot`. Find the Longhorn snapshot name inside the field `VolumeSnapshotContent.snapshotHandle`. Go to Longhorn UI. Delete the Longhorn snapshot. * Repeat steps in the section `Longhorn snapshot exist` above. PVC should be stuck in provisioning because Longhorn snapshot of the source volume doesn't exist. * Delete the PVC `test-restore-pvc` PVC #### Scenarios 3: Delete CSI snapshot * `longhorn-backup` type * Done in the above step * `longhorn-snapshot` type * volume is attached && snapshot doesn’t exist * Delete the VolumeSnapshot `test-snapshot-longhorn-snapshot` and verify that the VolumeSnapshot is deleted. * volume is attached && snapshot exist * Recreate the VolumeSnapshot `test-snapshot-longhorn-snapshot` * Verify the creation of Longhorn snapshot with the name in the field `VolumeSnapshotContent.snapshotHandle` * Delete the VolumeSnapshot `test-snapshot-longhorn-snapshot` * Verify that Longhorn snapshot is removed or marked as removed * Verify that the VolumeSnapshot `test-snapshot-longhorn-snapshot` is deleted. * volume is detached * Recreate the VolumeSnapshot `test-snapshot-longhorn-snapshot` * Scale down the workload to detach `test-vol` * Delete the VolumeSnapshot `test-snapshot-longhorn-snapshot` * Verify that VolumeSnapshot `test-snapshot-longhorn-snapshot` is stuck in deleting ### Upgrade strategy No upgrade strategy needed ## Note [optional] We need to update the docs and examples to reflect the new parameter in the VolumeSnapshotClass, `type`. ================================================ FILE: enhancements/20220317-snapshot-prune.md ================================================ # Snapshot Prune ## Summary Snapshot prune is a new snapshot-purge-related operation that helps **reclaim some space** from the snapshot file that is already marked as _Removed_ but **cannot be completely deleted**. This kind of snapshot is typically the one directly stands behind the volume head. ### Related Issues https://github.com/longhorn/longhorn/issues/3613 ## Motivation ### Goals Snapshots could store historical data for a volume. This means extra space will be required, and the volume actual size can be much greater than the spec size. To avoid existing volumes using too much space, users can clean up snapshots by marking the snapshots as _Removed_ then waiting for Longhorn purging them. But there is one issue: By design, the snapshot that directly stands behind the volume head, as known as the latest snapshot, cannot be purged by Longhorn after being marked as _Removed_. The space consumed by it cannot be released any matter if users care about historical data or not. Hence, Longhorn should do something special to reclaim space "wasted" by this kind of snapshot. ### Non-goals: Volume trim/shrink: https://github.com/longhorn/longhorn/issues/836 ## Proposal 1. Deleting a snapshot consists of 2 steps, marking the snapshot as _Removed_ then waiting for Longhorn purging it. And the snapshot purge consists of 3 steps: copy data from the newer snapshot to the old snapshot, replace the new snapshot with the updated old snapshot, remove the new snapshot. This operation is named "coalesce" or "fold" in Longhorn. As mentioned before, it cannot be applied to the latest snapshot file since the newer one of it is actually the volume head, which cannot be modified by others except for users/workloads. In other words, we cannot use this operation to handle the latest snapshot. ``` +--------------+ +--------------+ +--------------+ | Snapshot A | --- | Snapshot B | --- | Volume head | +--------------+ +--------------+ +--------------+ ^ | Marked Snapshot A (the old snapshot) as _Removed_ +--------------+ +--------------+ +--------------+ | Snapshot A | --- | Snapshot B | --- | Volume head | +--------------+ +--------------+ +--------------+ ^ | +---------------------+ Copy data from the Snapshot B (the newer snapshot) to Snapshot A +---------------------------------+ +--------------+ | Rename snapshot A to snapshot B | ----- | Volume head | +---------------------------------+ +--------------+ ^ | Delete Snapshot B then rename snapshot A to Snapshot B ``` 2. Longhorn needs to somehow reclaim the space from the latest snapshot without directly deleting the file itself or modifying the volume head. Notice that Longhorn can still read the volume head as well as modify the snapshot once the snapshot itself is marked as _Removed_. This means we can detect which part of the latest snapshot is overwritten by the volume head. Then punching holes in the overlapping parts of the snapshot would reclaim the space. Here, we call this new operation as "prune". ``` +--------------+ +---------------+ | Snapshot A | --- | Volume head | +--------------+ +---------------+ ^ | +---------------------+ Snapshot A is the latest snapshot of the volume. Longhorn will scan the volume head. For each data chunk of the volume head, Longhorn will punch a hole at the same position for snapshot A. ``` 3. Punching holes means modifying the data of the snapshot. Therefore, once the snapshot is marked as _Removed_ and the cleanup happens, Longhorn should not allow users to revert to the snapshot anymore. This is the prerequisite of this enhancement. This snapshot revert issue is handled in https://github.com/longhorn/longhorn/issues/3748. ### User Stories #### Cleanup the data of the latest snapshot Before the enhancement, users need to create a new snapshot, then remove the target snapshot so that Longhorn will coalesce the target snapshot with the newly created one. But the issue is, the volume head would be filled up later, and users may loop into redoing the operation to reclaim the space occupied by the historical data of the snapshot. After the enhancement, as long as there is no newer snapshot created, users can directly reclaim the space from the latest snapshot by simply deleting the snapshot via UI. ### User Experience In Detail Assume that there are heavy writing tasks for a volume and the only snapshot is filled up with the historical data (this snapshot may be created by rebuilding or backup). The actual size of the volume is typical twice the spec size. Now users just need to remove the only/latest snapshot via UI, Longhorn would reclaim almost all space used by the snapshot, which is the spec size here. Then as long as users don't create a new snapshot, the actual size of this volume is the space used by the volume head only, which is up to the spec size in total. ### API Changes N/A ## Design ### Implementation Overview #### longhorn-engine: When the snapshot purge is triggered, replicas will identify if the snapshot being removed is the latest snapshot by checking one child of it is the volume head. If YES, they will start the snapshot pruning operation: 1. Before pruning, replicas will make sure the apparent size of the snapshot is the same as that of the volume head. If No, we will truncate/expand the snapshot first. 2. During pruning, replicas need to iterate the volume head fiemap. Then as long as there is a data chunk found in the volume head file, they will blindly punch a hole at the same position of the snapshot file. If there are multiple snapshots including the latest one being removed simultaneously, we need to make sure the pruning is done only after all the other snapshots have done coalescing and deletion. #### longhorn-ui: Allow users to remove the snapshots that are already marked as Removed. And in this case, the frontend just needs to send a `SnapshotPurge` call to the backend. ### Test Plan #### Integration tests Test this snapshot prune operations with snapshot coalesce, snapshot revert, and volume expansion. ### Upgrade strategy N/A ================================================ FILE: enhancements/20220324-orphaned-data-cleanup.md ================================================ # Orphaned Replica Directory Cleanup ## Summary Orphaned replica directory cleanup identifies unmanaged replicas on the disks and provides a list of the orphaned replica directory on each node. Longhorn will not delete the replicas automatically, preventing deletions by mistake. Instead, it allows the user to select and trigger the deletion of the orphaned replica directory manually or deletes the orphaned replica directories automatically. ### Related Issues [https://github.com/longhorn/longhorn/issues/685](https://github.com/longhorn/longhorn/issues/685) ## Motivation ### Goals - Identify the orphaned replica directories - The scanning process should not stuck the reconciliation of the controller - Provide user a way to select and trigger the deletion of the orphaned replica directories - Support the global auto-deletion of orphaned replica directories ### Non-goals - Clean up unknown files or directories in disk paths - Support the per-node auto-deletion of orphaned replica directories - Support the auto-deletion of orphaned replica directories exceeded the TTL ## Proposal 1. Introduce a new CRD `orphan` and controller that represents and tracks the orphaned replica directories. The controller deletes the physical data and the resource if receive a deletion request. 2. The monitor on each node controller is created to periodically collects the on-disk replica directories, compares them with the scheduled replica, and then finds the orphaned replica directories. The reconciliation loop of the node controller gets the latest disk status and orphaned replica directories from the monitor and update the state of the node. Additionally, the `orphan` resources associated with the orphaned replica directories are created. ``` queue ┌───────────────┐ ┌──────────────────────┐ ┌┐ ┌┐ ┌┐ │ │ │ │ ... ││ ││ ││ ──────► │ syncNode() ├────────────►│ reconcile() │ └┘ └┘ └┘ │ │ │ │ └───────────────┘ └───────────┬──────────┘ │ syncWithMonitor │ │ ┌───────────▼──────────┐ │ │ │ per-node monitor │ │ | ┤ collect information │ │ │ └──────────────────────┘ ``` ### User Stories When a user introduces a disk into a Longhorn node, it may contain replica directories that are not tracked by the Longhorn system. The untracked replica directories may belong to other Longhorn clusters. Or, the replica CRs associated with the replica directories are removed after the node or the disk is down. When the node or the disk comes back, the corresponding replica data directories are no longer tracked by the Longhorn system. These replica data directories are called orphaned. Longhorn's disk capacity is taken up by the orphaned replica directories. Users need to compare the on-disk replica directories with the replicas tracked by the Longhorn system on each node and then manually delete the orphaned replica directories. The process is tedious and time-consuming for users. After the enhancement, Longhorn automatically finds out the orphaned replica directories on Longhorn nodes. Users can visualize and manage the orphaned replica directories via Longhorn GUI or command line tools. Additionally, Longhorn can deletes the orphaned replica directories automatically if users enable the global auto-deletion option. ### User Experience In Detail - Via Longhorn GUI - Users can check Node and Disk status then see if Longhorn already identifies orphaned replicas. - Users can choose the items in the orphaned replica directory list then clean up them. - Users can enable the global auto-deletion on setting page. By default, the auto-deletion is disabled. - Via `kubectl` - Users can list the orphaned replica directories by `kubectl -n longhorn-system get orphans`. - Users can delete the orphaned replica directories by `kubectl -n longhorn-system delete orphan `. - Users can enable the global auto-deletion by `kubectl -n longhorn-system edit settings orphan-auto-deletion` ## Design ### Implementation Overview **Settings** - Add setting `orphan-auto-deletion`. Default value is `false`. **Node controller** - Start the monitor during initialization. - Sync with the monitor in each reconcile loop. - Update the node/disk status. - Create `orphan` CRs based on the information collected by the monitor. - Delete the `orphan` CRs if the node/disk is requested to be evicted. - Delete the `orphan` CRs if the corresponding directories disappear. - Delete the `orphan` CRs if the auto-deletion setting is enabled. **Node monitor** - Struct ```go type NodeMonitor struct { logger logrus.FieldLogger ds *datastore.DataStore node longhorn.Node lock sync.RWMutex onDiskReplicaDirectories map[string][string]string syncCallback func(key string) ctx context.Context quit context.CancelFunc } ``` - Periodically detect and verify disk - Run `stat` - Check disk FSID - Check disk UUID in the metafile - Periodically check and identify orphan directories - List on-disk directories in `${disk_path}/replicas` and compare them with the last record stored in `monitor.onDiskDirectoriesInReplicas`. - If the two lists are different, iterate all directories in `${disk_path}/replicas` and then get the list of the orphaned replica directories. A valid replica directory has the properties: - The directory name format is `/replicas/-` - `/replicas/-/volume.meta` is parsible and follows the `volume.meta`'s format. - Compare the list of the orphaned replica directories with the `node.status.diskStatus.scheduledReplica` and find out the list of the orphaned replica directories. Store the list in `monitor.node.status.diskStatus.orphanedReplicaDirectoryNames` **Orphan controller** - Struct: ```go // OrphanSpec defines the desired state of the Longhorn orphaned data type OrphanSpec struct { // The node ID on which the controller is responsible to reconcile this orphan CR. // +optional NodeID string `json:"nodeID"` // The type of the orphaned data. // Can be "replica". // +optional Type OrphanType `json:"type"` // The parameters of the orphaned data // +optional // +nullable Parameters map[string]string `json:"parameters"` } // OrphanStatus defines the observed state of the Longhorn orphaned data type OrphanStatus struct { // +optional OwnerID string `json:"ownerID"` // +optional // +nullable Conditions []Condition `json:"conditions"` } ``` - If receive the deletion request, delete the on-disk orphaned replica directory and the `orphan` resource. - If the auto-deletion is enabled, node controller will issues the orphans deletion requests. **longhorn-ui** - Allow users to list the orphans on the node page by sending `OrphanList` call to the backend. - Allow users to select the orphans to be deleted. The frontend needs to send `OrphanDelete` call to the backend. ### Test Plan **Integration tests** - `orphan` CRs will be created correctly in the disk path. And they can be cleaned up with the directories. - `orphan` CRs will be created correctly when there are multiple kinds of files/directories in the disk path. And they can be cleaned up with the directories. - `orphan` CRs will be removed when the replica directories disappear. - `orphan` CRs will be removed when the node/disk is evicted or down. The associated orphaned replica directories should not be cleaned up. - Auto-deletion setting. ## Note[optional] ================================================ FILE: enhancements/20220408-support-kubernetes-ca.md ================================================ # Support Kubernetes Cluster Autoscaler Longhorn should support Kubernetes Cluster Autoscaler. ## Summary Currently, Longhorn pods are [blocking CA from removing a node](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node). This proposes to introduce a new global setting `kubernetes-cluster-autoscaler-enabled` that will annotate Longhorn components and also add logic for instance-manager PodDisruptionBudget management. ### Related Issues https://github.com/longhorn/longhorn/issues/2203 ## Motivation ### Goals - Longhorn should block CA from scaling down if a node met ANY condition: - Any volume attached - Contains a backing image manager pod - Contains a share manager pod - Longhorn should not block CA from scaling down if a node met ALL conditions: - All volume detached and there is another schedulable node with volume replica and replica IM PDB. - Not contain a backing image manager pod - Not contain a share manager pod ### Non-goals [optional] - CA setup. - CA blocked by kube-system components. - CA blocked by backing image manager pod. (TODO) - CA blocked by share manager pod. (TODO) ## Proposal Set `kubernetes-cluster-autoscaler-enabled` adds `cluster-autoscaler.kubernetes.io/safe-to-evict` annotation to Longhorn pods that are not backed by a controller, or with local storage volume mounts. To avoid data loss, Longhorn does not annotate the backing image manager and share manager pods. Currently, Longhorn creates instance-manager PDBs for replica/engine regardless of the volume state. During scale down, CA tries to find a removable node but failed by those instance-manager PDBs. We can add IM PDB handling to create and retained when the PDB is required: - There are volumes/engines running on the node. We need to guarantee that the volumes won't crash. - The only available/valid replica of a volume is on the node. Here we need to prevent the volume data from being lost. ### User Stories #### CA scaling Before the enhancement, CA will be blocked by - Pods that are not backed by a controller (engine/replica instance manager). - Pods with [local storage volume mounts](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/utils/drain/drain.go#L222) (longhorn-ui, longhorn-csi-plugin, csi-attacher, csi-provisioner, csi-resizer, csi-snapshotter). After enhancement, instance manager PDB will be actively managed by Longhorn: - Creates all engine/replica instance manager PDB when the volume is attached. - Delete engine instance manager PDB when the volume is detached. - Delete but keep 1 replica instance manager PDB when the volume is detached. the user can set a new global setting `kubernetes-cluster-autoscaler-enabled` to unblock CA scaling. This allows Longhorn to annotate Longhorn-managed deployments and engine/replica instance manager pods with `cluster-autoscaler.kubernetes.io/safe-to-evict`. ### User Experience In Detail - Configure the setting via Longhorn UI or kubectl. - Ensure all volume replica count is set to more than 1. - CA is not blocked by Longhorn components when the node doesn't contain volume replica, backing image manager pod, and share manager pod. - Engine/Replica instance-manager PDB will block the node if the volume is attached. - Replica instance-manager PDB will block the node when CA tries to delete the last node with the volume replica. ### API changes `None` ## Design ### Implementation Overview #### Global setting - Add new global setting `Kubernetes Cluster Autoscaler Enabled (Experimental)`. - The setting is `boolean`. - The default value is `false`. #### Annotations When setting `kubernetes-cluster-autoscaler-enabled` is `true`, Longhorn will add annotation `cluster-autoscaler.kubernetes.io/safe-to-evict` for the following pods: - The engine and replica instance-manager pods because those are not backed by a controller and use local storage mounts. - The deployment workloads are managed by the longhorn manager and using any local storage mount. The managed components are labeled with `longhorn.io/managed-by: longhorn-manager`. #### PodDisruptionBudget - No change to the logic to cleanup PDB if instance-manager doesn't exist. - Engine IM PDB: - Delete PDB if volumes are detached; - There is no instance process in IM (im.Status.Instance). - The same logic applies when a node is un-schedulable. Node is un-schedulable when marked in spec or with CA tainted `ToBeDeletedByClusterAutoscaler`; - Create PDB if volumes are attached; there are instance processes in IM (im.Status.Instance). - Replica IM PDB: - Delete PDB if setting `allow-node-drain-with-last-healthy-replica` is enabled. - Delete PDB if volumes are detached; - There is no instance process in IM (im.Status.Instance) - There are other schedulable nodes with healthy volume replica and have replica IM PDB. - Delete PDB when a node is un-schedulable. Node is un-schedulable when marked in spec or with CA tainted `ToBeDeletedByClusterAutoscaler`; - Check if the condition is met to delete PDB (same check as to when volumes are detached). - Enqueue the replica instance-manager of another schedulable node with the volume replica. - Delete PDB. - Create PDB if volumes are attached: - There are instance processes in IM (im.Status.Instance). - Create PDB when volumes are detached; - There is no instance process in IM (im.Status.Instance) - The replica has been started. There are no other schedulable nodes with healthy volume replica and have replica IM PDB. ### Test plan #### Scenario: test CA Given Cluster with Kubernetes cluster-autoscaler. And Longhorn installed. And Set `kubernetes-cluster-autoscaler-enabled` to `true`. And Create deployment with cpu request. ``` resources: limits: cpu: 300m memory: 30Mi requests: cpu: 150m memory: 15Mi ``` When Trigger CA to scale-up by increase deployment replicas. (double the node number, not including host node) ``` 10 * math.ceil(allocatable_millicpu/cpu_request*node_number/10) ``` Then Cluster should have double the node number. When Trigger CA to scale-down by decrease deployment replicas. (original node number) Then Cluster should have original node number. #### Scenario: test CA scale down all nodes containing volume replicas Given Cluster with Kubernetes cluster-autoscaler. And Longhorn installed. And Set `kubernetes-cluster-autoscaler-enabled` to `true`. And Create volume. And Attach the volume. And Write some data to volume. And Detach the volume. And Create deployment with cpu request. When Trigger CA to scale-up by increase deployment replicas. (double the node number, not including host node) Then Cluster should have double the node number. When Annotate new nodes with `cluster-autoscaler.kubernetes.io/scale-down-disabled`. (this ensures scale-down only the old nodes) And Trigger CA to scale-down by decrease deployment replicas. (original node number) Then Cluster should have original node number + 1 blocked node. When Attach the volume to a new node. This triggers replica rebuild. And Volume data should be the same. And Detach the volume. Then Cluster should have original node number. And Volume data should be the same. #### Scenario: test CA should block scale down of node running backing image manager pod Similar to `Scenario: test CA scale down all nodes containing volume replicas`. ### Upgrade strategy `N/A` ## Note [optional] `N/A` ================================================ FILE: enhancements/20220420-longhorn-snapshot-crd.md ================================================ # Longhorn Snapshot CRD ## Summary Supporting Longhorn snapshot CRD allows users to query/create/delete volume snapshots using kubectl. This is one step closer to making kubectl as Longhorn CLI. Also, this will be a building block for the future auto-attachment/auto-detachment refactoring for snapshot creation, deletion, volume cloning. ### Related Issues https://github.com/longhorn/longhorn/issues/3144 ## Motivation ### Goals 1. Support Longhorn snapshot CRD to allow users to query/create/delete volume snapshots using kubectl. 2. A building block for the future auto-attachment/auto-detachment refactoring for snapshot creation, deletion, volume cloning. 3. Pay attention to scalability problem. A cluster with 1k volumes might have 30k snapshots. We should make sure not to overload the controller work-queue as well as making too many grpc calls to engine processes. ## Proposal Introduce a new CRD, snapshot CRD and the snapshot controller. The life cycle of a snapshot CR is as below: 1. Create (by engine monitor/kubectl) 1. When user create a new snapshot CR, Longhorn try to create a new snapshot 2. When there is a snapshot in the volume that isn't corresponding to any snapshot CR, Longhorn will generate snapshot CR for that snapshot 2. Update (by snapshot controller) 1. Snapshot controller will reconcile the snapshot CR status with the snapshot info inside the volume engine 3. Delete (by engine monitor/kubectl) 1. When a snapshot CR is deleted (by user or by Longhorn), snapshot controller will make sure that the snapshot are removed from the engine before remove the finalizer and allow the deletion 2. Deleting volume should be blocked until all of its snapshot are removed 3. When there is a system generated snapshot CR that isn't corresponding to any snapshot info inside engine status, Longhorn will delete the snapshot CR ### User Stories Before this enhancement, users have to use Longhorn UI to query/create/delete volume snapshot. For user with only access to CLI, another option is to use our [Python client](https://longhorn.io/docs/1.2.4/references/longhorn-client-python/). However, the Python client are not as intuitive and easy as using kubectl. After this enhancement, users will be able to use kubectl to query/create/delete Longhorn snapshots just like what they can do with Longhorn backups. There is no additional requirement for users to use this feature. The experience details should be in the `User Experience In Detail` later. #### Story 1 User wants to limit the snapshot count to save space. Snapshot RecurringJobs set to Retain X number of snapshots do not touch unrelated snapshots, so if one ever changes the name of the RecurringJob, the old snapshots will stick around forever. These then have to be manually deleted in the UI. There might be some kind of browser automation framework might also work for pruning large numbers of snapshots, but this feels janky. Having a CRD for snapshots would greatly simplify this, as one could prune snapshots using kubectl, much like how one can currently manage backups using kubectl due to the existence of the `backups.longhorn.io` CRD. ### User Experience In Detail There is no additional requirement for users to use this feature. ### API changes We don't want to have disruptive changes in this initial version of snapshot CR (e.g., snapshot API create/delete shouldn't change. Snapshot status is still inside the engine status). We can wait for the snapshot CRD to be a bit more mature (no issue with scalability) and make the disruptive changes in the next version of snapshot CR (e.g., snapshot API create/delete changes to create/delete snapshot CRs. Snapshot status is removed from inside the engine status) ## Design ### Implementation Overview Introduce a new CRD, snapshot CRD and the snapshot controller. The snapshot CRD is: ```yaml // SnapshotSpec defines the desired state of Longhorn Snapshot type SnapshotSpec struct { // the volume that this snapshot belongs to. // This field is immutable after creation. // Required Volume string `json:"volume"` // require creating a new snapshot // +optional CreateSnapshot bool `json:"createSnapshot"` // The labels of snapshot // +optional // +nullable Labels map[string]string `json:"labels"` } // SnapshotStatus defines the observed state of Longhorn Snapshot type SnapshotStatus struct { // +optional Parent string `json:"parent"` // +optional // +nullable Children map[string]bool `json:"children"` // +optional MarkRemoved bool `json:"markRemoved"` // +optional UserCreated bool `json:"userCreated"` // +optional CreationTime string `json:"creationTime"` // +optional Size int64 `json:"size"` // +optional // +nullable Labels map[string]string `json:"labels"` // +optional OwnerID string `json:"ownerID"` // +optional Error string `json:"error,omitempty"` // +optional RestoreSize int64 `json:"restoreSize"` // +optional ReadyToUse bool `json:"readyToUse"` } ``` The life cycle of a snapshot CR is as below: 1. **Create** 1. When a snapshot CR is created, Longhorn mutation webhook will: 1. Add a volume label `longhornvolume: ` to the snapshot CR. This allow us to efficiently find snapshots corresponding to a volume without having listing potentially thoundsands of snapshots. 1. Add `longhornFinalizerKey` to snapshot CR to prevent it from being removed before Longhorn has change to clean up the corresponding snapshot 1. Populate the value for `snapshot.OwnerReferences` to uniquely identify the volume of this snapshot. This field contains the volume UID to uniquely identify the volume in case the old volume was deleted and a new volume was created with the same name. 2. For user created snapshot CR, the field `Spec.CreateSnapshot` should be set to `true` indicating that Longhorn should provision a new snapshot for this CR. 1. Longhorn snapshot controller will pick up this CR, check to see if there already is a snapshot inside the `engine.Status.Snapshots`. 1. If there is there already a snapshot inside engine.Status.Snapshots, update the snapshot.Status with the snapshot info inside `engine.Status.Snapshots` 2. If there isn't a snapshot inside `engine.Status.Snapshots` then: 1. making a call to engine process to check if there already a snapshot with the same name. This is to make sure we don't accidentally create 2 snapshots with the same name. This logic can be remove after [the issue](https://github.com/longhorn/longhorn/issues/3844) is resolved 1. If the snapshot doesn't inside the engine process, make another call to create the snapshot 3. For the snapshots that are already exist inside `engine.Status.Snapshots` but doesn't have corresponding snapshot CRs (i.e., system generated snapshots), the engine monitoring will generate snapshot CRs for them. The snapshot CR generated by engine monitoring with have `Spec.CreateSnapshot` set to `false`, Longhorn snapshot controller will not create a snapshot for those CRs. The snapshot controller only sync status for those snapshot CRs 2. **Update** 1. Snapshot CR spec and label are immutable after creation. It will be protected by the admission webhook 2. Sync the snapshot info from `engine.Status.Snapshots` to the `snapshot.Status`. 3. If there is any error or if the snapshot is marked as removed, set `snapshot.Status.ReadyToUse` to `false` 4. If there there is no snapshot info inside `engine.Status.Snapshots`, mark the `snapshot.Status.ReadyToUse` to `false`and populate the `snapshot.Status.Error` with the lost message. This snapshot will eventually be updated again when engine monitoring update `engine.Status.Snapshots` or it may be cleanup as the section below 4. **Delete** 1. Engine monitor will responsible for removing all snapshot CRs that don't have a matching snapshot info and are in one of the following cases: 1. The snapshot CRs with `Spec.CreateSnapshot: false` (snapshot CR that is auto generated by the engine monitoring) 2. The snapshot CRs with `Spec.CreateSnapshot: true` and `snapCR.Status.CreationTime != nil` (snapshot CR that has requested a new snapshot and the snapshot has already provisioned before but no longer exist now) 2. When a snapshot CR has deletion timestamp set, snapshot controller will: 1. Check to see if the actual snapshot inside engine process exist. 1. If it exist do: 1. if has not been marked as removed, issue grpc call to engine process to remove the snapshot 2. Check if the engine is in the purging state, if not issue a snapshot purge call to engine process 2. If it doesn't exist, remove the `longhornFinalizerKey` to allow the deletion of the snapshot CR ### Test plan Integration test plan. For engine enhancement, also requires engine integration test plan. ### Upgrade strategy Anything that requires if user want to upgrade to this enhancement ## Note [optional] How do we address scalability issue? 1. Controller workqueue 1. Disable resync period for snapshot informer 1. Enqueue snapshot only when: 1. There is a change in snapshot CR 1. There is a change in `engine.Status.CurrentState` (volume attach/detach event), `engine.Status.PurgeStatus` (for snapshot deletion event), `engine.Status.Snapshots` (for snapshot creation/update event) 1. This enhancement proposal doesn't make additional call to engine process comparing to the existing design. ## Todo For the special snapshot `volume-head`, we don't create a snapshot CR for this special snapshot because: 1. From the usecase perspective, user cannot delete this snapshot anyway so there is no need to generate this snapshot 1. The name `volume-head` is not globally uniquely, we might have to include volume name if we want to generate this snapshot CR 1. We would have to implement special logic to prevent user from deleting this special CR 1. On the flip side, if we generate this special CR, user will have a complete picture of the snapshot chain 2. The VolumeHead CR may suddenly point to another actual file during the snapshot creation. ================================================ FILE: enhancements/20220428-storage-network-through-grpc-proxy.md ================================================ # Storage Network Through gRPC Proxy ## Summary Currently, Longhorn uses the Kubernetes cluster CNI network and share the network with the entire cluster resources. This makes network availability impossible to control. We would like to have a global `Storage Network` setting to allow users to input an existing Multus `NetworkAttachmentDefinition` CR network in `/` format. Longhorn can use the storage network for in-cluster data traffics. The segregation can achieve by replacing the engine binary calls in the Longhorn manager with gRPC connections to the instance manager. Then the instance manager will be responsible for handling the requests between the management network and storage network. --- **_NOTE:_** There are other possible approaches we have considered to segregating the networks: - Add Longhorn Manager to the storage network. The Manager needs to restart itself to get the secondary storage network IP, and there is no storage network segregation to the Longhorn data plane (engine & replica). - Provide Engine/Replica with dual IPs. Code change around this approach is confusing and likely to increase maintenance complexity. --- ### Related Issues https://github.com/longhorn/longhorn/issues/2285 https://github.com/longhorn/longhorn/issues/3546 ## Motivation ### Goals - Have a new `Storage Network` setting. - Replace Manager engine binary calls with gRPC client to the instance manager. - Keep using the management network for the communication between Manager and Instance Manager. - Use the storage network for the data traffic of data plane components to the instance processes. Those are the engines and replicas in Instance Manager pods. - Support backward compatibility of the communication between the new Manager and the old Instance Manager after the upgrade. Ensure existing engine/replicas work without issues. ### Non-goals [optional] - Setup and configure the Multus `NetworkAttachmentDefinition` CRs. - Monitor for `NetworkAttachmentDefinition` CRs. The user needs to ensure the traffic is reachable between pods and across different nodes. Without monitoring, Longhorn will not get notified of the update of the `NetworkAttachmentDefinition` CRs. Thus the user should create a new `NetworkAttachmentDefinition` CR and update the `storage-network` setting. - Out-cluster data traffic. For example, backing image upload and download. ## Proposal ### Communication between Manager and Engine/Replica processes via Instance Manager gRPC proxy - Introduce a new gRPC server in Instance Manager. - Keep reusable connections between Manager and Instance Managers. - Allow Manager to fall back to engine binary call when communicating with old Instance Manager. ### Storage Network - Add a new `Storage Network` global setting. - Add `k8s.v1.cni.cncf.io/networks` annotation to pods that involve data transfer. The annotation will use the value from the storage network setting. Multus will attach a secondary network to pods with this annotation. - Engine instance manager pods - Replica instance manager pods - Backing image data source pods. Data traffic between replicas and backing image data source. - Backing image manager pods. Data traffic in-between backing image managers. - Add new `storageIP` to `Engine`, `Replica` and `BackingImageManager` CRD status. The storage IP will be use to communicate to the instance processes. ### User Stories #### Story 1 - set up the storage network As a Longhorn user / System administrator. I have set up Multus `NetworkAttachmentDefinition` for additional network management. And I want to segregate Longhorn in-cluster data traffic with an additional network interface. Longhorn should provide a setting to input the `NetworkAttachmentDefinition` CR name for the storage network. So I can guarantee network availability for Longhorn in-cluster data traffic. #### Story 2 - upgrade As a Longhorn user / System administrator. When I upgrade Longhorn, the changes should support existing attached volumes. So I can decide when to upgrade the Engine Image. ### User Experience In Detail #### Story 1 - set up the storage network 1. I have a Kubernetes cluster with Multus installed. 1. I created `NetworkAttachmentDefinition` CR and ensured the configuration is correct. 1. I Added `/` to Longhorn `Storage Network` setting. 1. I see setting update failed when volumes are attached. 1. I detach all volumes. 1. When updating the setting I see engine/replica instance manager pod and backing image manager pods is restarted. 1. I attach the volumes. 1. I describe Engine, Replica, and BackingImageManager, and see the `storageIP` in CR status is in the range of the `NetworkAttachmentDefinition` subnet/CIDR. I also see the `storageIP` is different from the `ip` in CR status. 1. I describe the Engine and see the `replicaAddressMap` in CR spec and status is using the storage IP. 1. I see pod logs indicate the network directions. #### Story 2 - upgrade 1. I Longhorn v1.2.4 cluster. 1. I have healthy volumes attached. 1. I upgrade Longhorn. 1. I see volumes still attached and healthy with available engine image upgrade. 1. I cannot upgrade the volume engine image with the volume attached. 1. After I detach the volume, I can upgrade its engine image. 1. I attached the volumes. 1. I see the volumes are healthy. ### API changes - The new global setting `Storage Network` will use the existing `/v1/settings` API. ## Design ### Overview gRPC Proxy Implementation #### Instance Manager - Start the gRPC proxy server with the next port to the process server. The default should be `localhost:8501`. - The gRPC proxy service shares the same `imrpc` package name as the process server. ``` Ping ServerVersionGet VolumeGet VolumeExpand VolumeFrontendStart VolumeFrontendShutdown VolumeSnapshot SnapshotList SnapshotRevert SnapshotPurge SnapshotPurgeStatus SnapshotClone SnapshotCloneStatus SnapshotRemove SnapshotBackup SnapshotBackupStatus BackupRestore BackupRestoreStatus BackupVolumeList BackupVolumeGet BackupGet BackupConfigMetaGet BackupRemove ReplicaAdd ReplicaList ReplicaRebuildingStatus ReplicaVerifyRebuild ReplicaRemove ``` #### Manager - Create a _proxyHandler_ object to map the controller ID to an _EngineClient_ interface. The _proxyHandler_ object is shared between controllers. - The Instance Manager Controller is responsible for the life cycle of the proxy gRPC client. For every enqueue: - Check for the existing gRPC client in the _proxyHandler_, and check the connection liveness with the `Ping` request. - If the proxy gRPC client connection is dead, stop the proxy gRPC client and error so it will re-queue. - If the proxy gRPC client doesn't exist in the _proxyHandler_, start a new gRPC connection and map it to the current controller ID. - Do not create the proxy gRPC connection when the instance manager version is less than the current version. We will provide the fallback interface caller provided when getting the client. - The gRPC client will use the _EngineClient_ interface. - Provide a fallback interface caller when getting the gRPC client from the _proxyHandler_. The fallback callers are: - the existing `Engine` client used for the binary call - `BackupTargetClient`. - Use the fallback caller when the instance manager version is less than the current version. - Add new `BackupTargetBinaryClient` interface for fallback. ``` type BackupTargetBinaryClient interface { BackupGet(destURL string, credential map[string]string) (*Backup, error) BackupVolumeGet(destURL string, credential map[string]string) (volume *BackupVolume, err error) BackupNameList(destURL, volumeName string, credential map[string]string) (names []string, err error) BackupVolumeNameList(destURL string, credential map[string]string) (names []string, err error) BackupDelete(destURL string, credential map[string]string) (err error) BackupVolumeDelete(destURL, volumeName string, credential map[string]string) (err error) BackupConfigMetaGet(destURL string, credential map[string]string) (*ConfigMetadata, error) } ``` - Introduce A new `EngineClientProxy` interface for the Proxy, which includes proxy-specific methods and implementation of the existing `EnglineClient` and `BackupTargetClient` interfaces. This will be adaptive when using the EngineClient interface for the proxy or non-proxy/fallback operations. ``` type EngineClientProxy interface { EngineClient BackupTargetBinaryClient IsGRPC() bool Start(*longhorn.InstanceManager, logrus.FieldLogger, *datastore.DataStore) error Stop(*longhorn.InstanceManager) error Ping() error } ``` ### Overview Storage Network Overview Implementation #### Setting Add a new global setting `Storage Network`. - The setting is `string`. - The default value is `""`. - The setting should be in the `danger zone` category. - The setting will be validated at admission webhook setting validator. - The setting should be in the form of `< NAMESPACE>/`. - The setting cannot be updated when volumes are attached. #### CRD Engine: - New `storageIP` in status. - Use the replica `status.storageIP` instead of the replica `status.IP` for the replicaAddressMap. Replica: - New `storageIP` in status. BackingImageManager: - New `storageIP` in status. #### Instance Manager Controller 1. When creating instance manager pods, add `k8s.v1.cni.cncf.io/networks` annotation with `lhnet1` as interface name. Use the `storage-network` setting value for the namespace and name. ``` k8s.v1.cni.cncf.io/networks: ' [ { "namespace": "kube-system", "name": "demo-10-30-0-0", "interface": "lhnet1" } ] ' ``` #### Instance Handler 1. Get the IP from instance manager Pod annotation `k8s.v1.cni.cncf.io/network-status`. Use the IP for `Engine` and `Replica` Storage IP. When the `storage-network` setting is empty, The Storage IP will be the pod IP. #### Backing Image Manager Controller 1. When creating backing image manager pods, add `k8s.v1.cni.cncf.io/networks` annotation with `lhnet1` as interface name. Use the `storage-network` setting value for the namespace and name. ``` k8s.v1.cni.cncf.io/networks: ' [ { "namespace": "kube-system", "name": "demo-10-30-0-0", "interface": "lhnet1" } ] ' ``` 1. Get the IP from backing image manager Pod annotation `k8s.v1.cni.cncf.io/network-status`. Use the IP for `BackingImageManager` Storage IP. When the `storage-network` setting is empty, The Storage IP will be the pod IP. #### Backing Image Data Source Controller 1. When creating backing image data source pods, add `k8s.v1.cni.cncf.io/networks` annotation with `lhnet1` as interface name. Use the `storage-network` setting value for the namespace and name. ``` k8s.v1.cni.cncf.io/networks: ' [ { "namespace": "kube-system", "name": "demo-10-30-0-0", "interface": "lhnet1" } ] ' ``` #### Backing Image Manager - Export From volume 1. get the IPv4 of the `lhnet1` interface and use it as the receiver address. Use the pod IP if the interface doesn't exist. #### Setting Controller 1. Do not update the `storage-network` setting and return an error when `Volumes` are attached. 1. Delete all backing image manager pods. 1. Delete all instance manager pods. ### Test plan #### CI Pipeline All existing tests should pass when the cluster has the storage network configured. We should consider having a new test pipeline for the storage network. Infra Prerequisites: - Secondary network interface added to each cluster instance. - Multus deployed. - Network-attachment-definition created. - Routing is configured in all cluster nodes to ensure the network is accessible between instances. - For AWS, disable network source/destination checks for each cloud-provider instance. #### Test storage-network setting Scenario: `Engine`, `Replica` and `BackingImageManager` should use IP in `storage-network` `NetworkAttachmentDefinition` subnet/CIDR range after setting update. ### Upgrade strategy [Some old instance manager pods are still running after upgrade](https://longhorn.io/kb/troubleshooting-some-old-instance-manager-pods-are-still-running-after-upgrade/). Old engine instance managers do not have the gRPC proxy server for Manager to communicate. Hence, we need to support backward compatibility. Manager communication: - Bump instance manager API version. - Manager checks for incompatible version and fall back to requests through the engine binary. Volume/Engine live upgrade: - Keep live upgrade. This will be a soft notice for users to know we will not enforce any change in 1.3, but it will happen in 1.4. ## Note [optional] `None` ================================================ FILE: enhancements/20220727-dedicated-recovery-backend-for-rwx-volume-nfs-server.md ================================================ # Dedicated Recovery Backend for RWX Volume's NFS Server ## Summary A NFS server located within the share-manager pod is a key component of a RWX volume. The share-manager controller will recreate the share-manager pod and attach the volume to another node while the node where the share-manager pod is running is down. However, the failover cannot work correctly because the NFS server lacks an appropriate recovery backend and the connection information cannot be persistent. As a result, the client's workload will be interrupted during the failover. To make the NFS server have failover capability, we want to implement a dedicated recovery backend and associated modifications for Longhorn. ### Related Issues [https://github.com/longhorn/longhorn/issues/2293](https://github.com/longhorn/longhorn/issues/2293) ## Motivation ### Goals - Implement a dedicated recovery backend for Longhorn and make the NFS server highly available. ### Non-goals - Active/active or Active/passive NFS server pattern ## Proposal To support NFS server's failover capability, we need to change both the client and server configurations. A dedicated recovery backend for Kubernetes and Longhorn is also necessary. In the implementation, we will not implement the active/active or active/passive server pattern. Longhorn currently supports local filesystems such as ext4 and xfs. Thus, any change in the node, which is providing service, cannot update to the standby node. The limitation will hinder the active/active design. Currently, the creation of an engine process needs at least one replica and then exports the iSCSI frontend. That is, the standby engine process of the active/passive configuration is not allowable in current Longhorn architecture. ### User Stories While the node where the share-manager pod is running is down, the share-manager controller will recreate the share-manager pod and attach the volume to another node. However, the failover cannot work correctly because the connection information are lost after restarting the NFS server. As a result, the locks cannot be reclaimed correctly, and the interruptions of the clients’ filesystem operations happen. ### User Experience In Detail 1. The changes and improvements should not impact the usage of the RWX volumes. 2. NFS filesystem operation will not be interrupted after the failover of the share-manager pod. 3. After the crash of the share-manager pod, the application on the client side’s IO operations will be stuck until the share-manager and NFS are recreated. 4. To make the improvement work, users have to make sure the hostname of each node in the Longhorn system is unique by checking each node's hostname using `hostname` command. 5. To shorten the failover time, users can - Multiple coredns pods in the Kubernetes cluster to ensure the recovery backend be always accessible. - Reduce the NFS server's `Grace_Period` and `Lease_Lifetime`. By default, `Grace_Period` and `Lease_Lifetime` are 90 and 60 seconds, respectively. However, the value can be reduced to a smaller value for early termination of the grace period at the expense of security. Please refer to [Long client timeouts when failing over the NFS Ganesha IP resource](https://www.suse.com/support/kb/doc/?id=000019374). - Reduce `node-monitor-period` and `node-monitor-grace-period` values in the Kubelet. The unresponsive node will be marked as `NotReady` and speed up the NFS server's failover process. ## Design ### Implementation Overview - **longhorn-manager** The speed of a share-manager pod and volume's failover is affected by the cluster's settings and resources, so it is unpredictable how long it takes to failover. Thus, the NFS client mount options `soft, timeo=30, retrans=3` are replaced with `hard`. - **share-manager** To allow the NFSv4 clients to reclaim locks after the failover of the NFS server, the grace period is enabled by setting - Lease_Lifetime = 60 - Grace_Period = 90 Additionally, set `NFS_Core_Param.Clustered` to `false`. The NFS server will use the hostname rather than such as `node0` in the share-manager pod, which is same as the name of the share-manager pod, to create a corresponding storage in the recovery backend. The unique hostname avoids the naming conflict in the recovery backend. - **nfs-ganesha (user-space NFS server)** ``` ┌────────────────────────────────────────────────┐ │ service │ ┌──► │ │ │ longhorn-nfs-recovery-backend │ │ └───────────────────────┬────────────────────────┘ │ │ HTTP API ┌─────────────┴──────────────┐ │ │ │ │ │ endpoint 1 │ endpoint N ┌──────────────────────┐ │ ┌─────────▼────────┐ ┌────────▼─────────┐ │ share-manager pod │ │ │ recovery-backend │ │ recovery-backend │ │ │ │ │ pod │ │ pod │ │ ┌──────────────────┐ │ │ │ │ ... │ │ │ │ nfs server ├─┼─┘ │ │ │ │ │ └──────────────────┘ │ │ │ │ │ │ │ │ │ │ │ └──────────────────────┘ └──────────┬───────┘ └──────────┬───────┘ │ │ │ ┌─────────────┐ │ └───────►│ configMap │◄─────┘ └─────────────┘ ``` 1. Introduce a recovery-backend service backed by multiple recovery-backend pods. The recover-backend is shared by multiple RWX volumes to reduce the costs of the resources. 2. Implement a set of dedicating recovery-backend operations for Longhorn in nfs-ganesha - recovery_init - Create a configmap, `recovery-backend-${share-manager-pod-name}`, storing the client information - end_grace - Clean up the configmap - recovery_read_clids - Create the client reclaim list from the configmap - add_clid - Add the client key (client’s hostname) into the configmap - rm_clid - Remove the client key (client’s hostname) from the configmap - add_revoke_fh - Revoke the delegation 3. Then, the data from the above operations are persisted by sending to the recovery-backend service. The data will be saved in the configmap, `recovery-backend-${share-manager-pod-name}`. - **Dedicating Configmap Format** ``` name: `recovery-backend-${share-manager-pod-name}` labels: longhorn.io/component: nfs-recovery-backend ... annotations: version: 8-bytes random id, e.g. 6SVVI1LE data: 6SVVI1LE: {….json encoded content (containing the client identity information…} ``` One example ``` apiVersion: v1 data: 6SVVI1LE: '{"31:Linux NFSv4.1 rancher50-worker1":[],"31:Linux NFSv4.1 rancher50-worker2":[],"31:Linux NFSv4.1 rancher50-worker3":[]}' kind: ConfigMap metadata: annotations: version: 6SVVI1LE creationTimestamp: "2022-12-01T01:27:14Z" labels: longhorn.io/component: share-manager-configmap longhorn.io/managed-by: longhorn-manager longhorn.io/share-manager: pvc-de201ca5-ec0b-42ea-9501-253a7935fc3e name: recovery-backend-share-manager-pvc-de201ca5-ec0b-42ea-9501-253a7935fc3e namespace: longhorn-system resourceVersion: "47544" uid: 60e29c30-38b8-4986-947b-68384fcbb9ef ``` ### Notice - **In the event that the original share manager pod is unavailable, a new share manager pod cannot be created** In the client side, IO to the RWX volume will hang until a share-manager pod replacement is successfully created on another node. - **Failed to reclaim locks in 90-seconds grace period** If locks cannot be reclaimed after a grace period, the locks are discarded and return IO errors to the client. The client reestablishes a new lock. The application should handle the IO error. Nevertheless, not all applications can handle IO errors due to their implementation. Thus, it may result in the failure of the IO operation and the loss of data. Data consistency may be an issue. - **If the DNS service goes down, share-manager pod will not be able to communicate with longhorn-nfs-recovery-backend** The NFS-ganesha server in the share-manager pod communicates with longhorn-nfs-recovery-backend via the service `longhorn-recovery-backend` IP. Thus, the high availability of the DNS services is recommended for avoiding the communication failure. ### Test Plan - Setup 3 worker nodes for the Longhorn cluster - Attach 1 RWO volume to node-1 - Attach 2 RWO volumes to node-2 - Attach 3 RWO volumes to node-3 - Tests 1. Create 1 RWX volume and then run an app pod with the RWX volume on each worker node.Execute the command in each app pod `( exec 7<>/data/testfile-${i}; flock -x 7; while date | dd conv=fsync >&7 ; do sleep 1; done )` where ${i} is the node number. Turn off the node where share-manager is running. Once the share-manager pod is recreated on a different node, check - Expect - In the client side, IO to the RWX volume will hang until a share-manager pod replacement is successfully created on another node. - During the grace period, the server rejects READ and WRITE operations and non-reclaim locking requests (i.e., other LOCK and OPEN operations) with an error of NFS4ERR_GRACE. - The clients can continue working without IO error. - Lock reclaim process can be finished earlier than the 90-seconds grace period. - During the grace period, the server reject READ and WRITE operations and non-reclaim - If locks cannot be reclaimed after a grace period, the locks are discarded and return IO errors to the client. The client reestablishes a new lock. 2. Turn the deployment into a daemonset in [example]([https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml](https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml) ) and disable `Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly`. Then, deploy the daemonset with a RWX volume. Turn off the node where share-manager is running. Once the share-manager pod is recreated on a different node, check - Expect - The other active clients should not run into the stale handle errors after the failover. - Lock reclaim process can be finished earlier than the 90-seconds grace period. 3. Multiple locks one single file tested by byte-range file locking Each client ([range_locking.c](https://github.com/longhorn/longhorn/files/9208112/range_locking.txt)) in each app pod locks a different range of the same file. Afterwards, it writes data repeatedly into the file. Turn off the node where share-manager is running. Once the share-manager pod is recreated on a different node, check - The clients continue the tasks after the server's failover without IO or stale handle errors. - Lock reclaim process can be finished earlier than the 90-seconds grace period. ## Note[optional] ### Reference for the NFSv4 implementation - [Network File System (NFS) Version 4 Protocol](https://datatracker.ietf.org/doc/html/rfc7530) - [Long client timeouts when failing over the NFS Ganesha IP resource](https://www.suse.com/support/kb/doc/?id=000019374) - [Necessary NFS Server Cluster Design for NFS Client Lock Preservation](https://www.suse.com/support/kb/doc/?id=000020396) - [How NFSv4 file delegations work](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-DE6FECB5-FA4D-4957-BA68-4B8822EF8B43.html) ================================================ FILE: enhancements/20220801-failed-backups-cleanup.md ================================================ # Failed Backup Clean Up ## Summary Longhorn will leave the failed backups behind and will not delete the backups automatically either until the backup target is removed. Failed backup cleanup will be occurred when making a backup to remote backup target failed. This LEP will trigger the deletion of failed backups automatically. ### Related Issues [[IMPROVEMENT] Support failed/obsolete orphaned backup cleanup](https://github.com/longhorn/longhorn/issues/3898) ## Motivation ### Goals - Support the auto-deletion of failed backups that exceeded the TTL. - Support the global auto-deletion option of failed backups cleanup for users. - The process should not be stuck in the reconciliation of the controllers. ### Non-goals [optional] - Clean up unknown files or directories on the remote backup target. ## Proposal 1. The `backup_volume_controller` will be responsible for deleting Backup CR when there is a backup which state is in `Error` or `Unknown`. The reconciliation procedure of the `backup_volume_controller` gets the latest failed backups from the datastore and delete the failed backups. ```text queue ┌───────────────┐ ┌┐ ┌┐ ┌┐ │ │ ... ││ ││ ││ ──────► │ syncHandler() | └┘ └┘ └┘ │ │ └───────┬───────┘ │ ┌──────────▼───────────┐ │ │ │ reconcile() | │ │ └──────────┬───────────┘ │ ┌──────────▼───────────┐ │ │ │ get failed backups │ │ | | then delete them │ │ │ └──────────────────────┘ ``` ### User Stories When a user or recurring job tries to make a backup and store it in the remote backup target, many situations will cause the backup procedure failed. In some cases, there will be some failed backups still staying in the Longhorn system and this kind of backups are not handled by the Longhorn system until user removes the backup target. Or users can manage the failed backups via Longhorn GUI or command line tools manually. After the enhancement, Longhorn can delete the failed backups automatically after enabling auto-deletion. ### User Experience In Detail - Via Longhorn GUI - Users can be aware of that backup was failed if auto-deletion is disabled. - Users can check the event log to understand why the backup failed and deleted. - Via `kubectl` - Users can list the failed backups by `kubectl -n longhorn-system get backups` if auto-deletion is disabled. ## Design ### Implementation Overview **Settings** - Add setting `failed-backup-ttl`. Default value is `1440` minutes and set to `0` to disable the auto-deletion. **Failed Backup** - Backups in the state `longhorn.BackupStateError` or `longhorn.BackupStateUnknown`. **Backup Controller** - Start the monitor and sync the backup status with the monitor in each reconcile loop. - Update the backup status. - Trigger `backup_volume_controller` to delete the failed backups. **Backup Volume controller** - Reconcile loop usually is triggered after backupstore polling which is controlled by **Backupstore Poll Interval** setting. - Start to get all backups in each reconcile loop - Tell failed backups from all backups and try to delete failed backups by default. - Update the backup volume CR status. ### Test plan **Integration tests** - `backups` CRs with `Error` or `Unknown` state will be removed by `backup_volume_controller` triggered by backupstore polling when the `backup_monitor` detects the backup failed. - `backups` CRs with `Error` or `Unknown` state will not be handled if the auto-deletion is disabled. ## Note [optional] ### Why not leverage the current orphan framework 1. We already have the backup CR to handle the backup resources and failed backup is not like orphaned replica which is not owned by any volume at the beginning. 2. Cascading deletion of orphaned CR and backup CR would be more complicated than we just handle the failed backups immediately when backup procedure failed. Both in this LEP or orphan framework we would delete the failed backups by `backup_volume_controller`. 3. Listing orphaned backups and failed backups on both two UI pages `Orphaned Data` and `Backup` might be a bit confusing for users. Deleting items manually on either of two pages would be involved in what it mentioned at statement 2. ================================================ FILE: enhancements/20220913-longhorn-system-backup-restore.md ================================================ # Longhorn System Backup/Restore ## Summary This feature is to support the Longhorn system backup and restore. And also allows the user to rollback the Longhorn system to the previous healthy state after a failed upgrade. Currently, we have documents to guide users on how to restore Longhorn: - [Restore to a new cluster using Velero](https://longhorn.io/docs/1.3.0/advanced-resources/cluster-restore/restore-to-a-new-cluster-using-velero/) - [Restore to a cluster contains data using Rancher snapshot](https://longhorn.io/docs/1.3.0/advanced-resources/cluster-restore/restore-to-a-cluster-contains-data-using-rancher-snapshot/) However, the solution relies on third-party tools, not out-of-the-box, and involves tedious human intervention. With this new feature, Longhorn's custom resources will be backed up and bundled into a single system backup file, then saved to the remote backup target. Later, users can choose a system backup to restore to a new cluster or restore to an existing cluster; nevertheless, this allows for cluster rollback to fix the corrupted cluster state just after a failed upgrade. ### Related Issues https://github.com/longhorn/longhorn/issues/1455 ## Motivation ### Goals - Support Longhorn system backup by backing up Longhorn custom resources, bundling them to a single file, and uploading it to the backup target. - Support Longhorn system restore to a new/existing cluster from a system backup. - Support Longhorn system restoration to the previous healthy state when encountering the failed upgrade. - Support restoring volume from the `lastBackup` when the volume doesn't exist in the cluster during the system restore. - Support not restoring volume from the `lastBackup` when the volume exists in the cluster during the system restore. ### Non-goals [optional] - This feature does not deploy the Longhorn cluster. Users need to have a running Longhorn cluster to run the system restore. - Do backup/restore the downstream workloads attached to Longhorn volumes. For example, if a cluster has a Pod with a PersistentVolumeClaim. Longhorn will restore the PersistentVolumeClaim, PersistentVolume, and Volume only. After a successful system restoration, the user can re-deploy the pod with the same manifest. - Restore while there are volumes still attached. - Automatically create a system backup before an upgrade and run the system restore when the Longhorn upgrade fails. Instead, the user needs to run a system backup and restore it manually. - Delete resources none-existing in the system backup. We can probably enhance it later and make this an option for the users. ## Proposal ### System Backup ``` |------------ [[ longhorn-backup-target ]] ----------| | controller | | | v v [ SystemBackup ] ---> [[ longhorn-system-backup ]] ---> [ object store ] CR controller | | backupstore/system-backups/// | [ system-backups.zip ] <--|--> [ system-backup.cfg ] ``` 1. Introduce new [v1/systembackups](#longhorn-manager-http-api) HTTP APIs. 1. Introduce a new [SystemBackup](#manager-systembackup-custom-resource) custom resource definition. - A new custom resource triggers the creation of the system [resource](#system-backup-resources) backup. - Deleting the custom resource triggers deletion of the system backup in the backup target. This behavior is similar to the current backup resource handling. - The system backups stored in the backup target will get synced to the custom resources list. 1. Introduce new responsibility to `longhorn-backup-target` controller. - Responsible for syncing system backups in the backup target to the `SystemBackup` list. 1. Introduce a new `longhorn-system-backup` controller. - Responsible for generating the system backup file, bundling them to a single file, and uploading it to the backup target. 1. Generates the Longhorn resources YAML files. 1. Compress resources YAML files to a zip file. 1. Upload to the backup target `backupstore/system-backups//`. >**Note:** Do not create/upload the system backup when the `SystemBackup` is created by the backup target controller. - Responsible for deleting system backup in the backup target. - Responsible for updating `SystemBackup` status. - Responsible for updating the error message in the `SystemBackup` status condition. Reference [SystemRestore](#manager-systemrestore-custom-resource) condition as the example. 1. Introduce `SystemBackup` webhook validator for [condition validation](#validator-system-backup). ### System Restore ``` [ system-backup.zip ] [ backups ] ^ ^ | | [ system-backup.cfg ] <-- backupstore/system-backups/... | | | | backupstore/volumes/ | backupstore/backing-images/ | | | | [ SystemRestore ] ---> [[ longhorn-system-restore ]] ---> [ object-store ] CR controller | | | | | | | | [ Job ] | [ system-backup.cfg: engine ] | | | v | | [ system-backup.cfg: manager ] ---> [[ longhorn-system-rollout ]] controller | | | v | <--- [ Volume: from backup ] | <--- [ BackingImage: from backup] V [ Resources ] ``` 1. Introduce new [v1/systemrestores](#longhorn-manager-http-api) HTTP APIs. 1. Introduce a new [SystemRestore](#manager-systemrestore-custom-resource) custom resource definition. - A new custom resource triggers the creation of a system restore job. - Deleting the custom resource triggers the deletion of the system restore job. 1. Introduce a new `longhorn-system-restore` controller. - Responsible for creating a new system restore job that runs a `longhorn-system-rollout` controller. - Responsible for deleting the system restore job. 1. Introduce a new `longhorn-system-rollout` controller. This controller is similar to the `uninstall controller`. - Run inside the pod created by the system restore job. - Responsible for downloading the system-backup from the backup target. - Responsible for restoring [resources](#system-backup-resources) from the system-backup file. - Responsible for updating `SystemRestore` status during system restore. - Responsible for adding `longhorn.io/last-system-restore`, and `longhorn.io/last-system-restore-at` annotation to the restored resources. - Responsible for adding `longhorn.io/last-system-restore-backup`annotation to the restored volume resources. - Responsible for updating the error message in the [SystemRestore](#manager-systemrestore-custom-resource) status condition. > **Note:** There are 2 areas covered for cross version restoration: > 1. The system restore will use the manager and engine image in the system backup config to run the `longhorn-system-rollout` so the controller is compatible with the restoring resources. > 1. When the CustomResourceDefinition is missing the version for the restoring resource, the system restore doesn't replace or remove the existing CustomResourceDefinitions. Instead, the controller adds to its versions. So system restoration doesn't break existing resources. > > See [Specify multiple versions](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#specify-multiple-versions) for details. 1. Introduce `SystemRestore` webhook validator for [condition validation](#validator-system-restore). ### User Stories #### Longhorn system restore Before the enhancement, the user can follow the [documents](#summary) to leverage external solutions to backup and restore the Longhorn system. After the enhancement, the user can backup/restore Longhorn system to/from the backup target using this Longhorn native solution. #### Upgrade rollback (downgrade) Before the enhancement, the user cannot downgrade Longhorn. After the enhancement, the user can downgrade Longhorn when there is a pre-upgrade system backup in the backup target. > **Note:** For the Longhorn cluster version before v1.4.0, users still need to follow the Longhorn [documents](#summary) for backup and restore the Longhorn system. ### User Experience In Detail ### Longhorn UI 1. Go to `System Backup` from the Setting drop-down menu. ``` | Dashboard | Node | Volume | Recurring Job | Backup | Setting v | + ======================= + | General | | Engine Image | | Orphaned Data | | Backing Image | | Instance Manager Image | | System Backup | + ======================= + ``` 1. View `System Backups` and `System Restores` on the same page. ``` System Backups [Custom Column] ==================================================================================================================== [Create] [Delete] [Restore] [Search Box v ][__________][Go] + ======= + ======================================================================================| Name |=================== [] | Version | Name | State | Error | State | ---+----------+--------+--------+-----------------------------------------------------| Version |------------------- [] | 1.4.0 | demo-1 | Error | error uploading system backup: failed to execute: + ======= + /engine-binaries/c3y1huang-research-000-lh-ei/longhorn [system-backup : : : : upload --source /tmp/demo-2.zip --dest s3://backupbucket@us-east-1/ --name demo-2 --manager-image c3y1huang/research:000-lh-manager : : : : --engine-image c3y1huang/research:000-lh-ei], output , stderr, time=\"2022-08-16T03:52:09Z\" level=fatal msg=\"Failed to run upload : : : : system-backup command\" error=\"missing required parameter --longhorn-version\"\n, error exit status 1 [] | 1.4.0 | demo-2 | Ready | ==================================================================================================================== [<] [1] [>] System Restores [Custom Column] ==================================================================================================================== [Delete] [Search Box v ][_________][Go] + ======= + ====================================================================================== | Name |================== [] | Name | Version | State | Age | Error | State | ---+----------------+-----------+--------------+-------+------------------------------ | Version |------------------ [] | demo-1-restore | v1.4.0 | Completed | 2m26s | + ======= + [] | demo-2-foobar | v1.4.0 | Error | 64s | Download: sample error message [] | demo-2-restore | v1.4.0 | Initializing | 1s | =================================================================================================================== [<] [1] [>] ``` ### System Backup ***Longhorn GUI*** - The user can create system backups to the backup target. - The system backup will be uploaded to backup target `backupstore/system-backups//`. - The user can view the system backup status. ***Command kubectl*** - The user can create `SystemBackup` to backup Longhorn system to the backup target. ```yaml apiVersion: longhorn.io/v1beta2 kind: SystemBackup metadata: name: demo-2 namespace: longhorn-system ``` - The user can view the system backups. ``` > kubectl -n longhorn-system get lhsb NAMESPACE NAME VERSION STATE CREATED longhorn-system demo-1 v1.4.0 Error 2022-08-23T00:25:29Z longhorn-system demo-2 v1.4.0 Ready 2022-08-24T02:34:57Z ``` ### System Restore ***Longhorn GUI*** - The users can restore a system backup in the backup target. - The users can view the system restore status. - The users can restore from a different Longhorn version. ***Command kubectl*** - The user can create `SystemRestore` to restore system backup in the backup target. ```yaml apiVersion: longhorn.io/v1beta2 kind: SystemRestore metadata: name: demo-2-restore namespace: longhorn-system spec: systemBackup: demo-2 ``` - The users can view the system restores. ``` > kubectl -n longhorn-system get lhsr NAME STATE AGE demo-1-restore Completed 2m26s demo-2-foobar Error 64s demo-2-restore Initializing 1s ``` ### API changes #### Longhorn manager HTTP API | Method | Path | Description | | ---------- | -------------------------------- | ---------------------------------------------------------------------- | | **POST** | `/v1/systembackups` | Generates system backup file and upload to the backup target | | **GET** | `/v1/systembackups` | Get all system backups. Including ones already exist in the backup target and ones that are initialized but do not exist in the backup target | | **DELETE** | `/v1/systembackups/{name}` | Delete system backup saved in the backup target | | **POST** | `/v1/systemrestores` | Download a system backup from the backup target and restore it | | **GET** | `/v1/systemrestores` | Get all system restores | | **DELETE** | `/v1/systemrestores/{name}` | Delete a system restore | | | `/v1/ws/{period}/systembackups` | Websocket stream for system backups | | | `/v1/ws/{period}/systemrestores` | Websocket stream for system restores | ## Design ### Implementation Overview #### Manager: SystemBackup custom resource ```yaml apiVersion: longhorn.io/v1beta2 kind: SystemBackup metadata: creationTimestamp: "2022-08-25T02:50:06Z" finalizers: - longhorn.io generation: 1 labels: longhorn.io/version: v1.4.0 name: demo-2 namespace: longhorn-system resourceVersion: "420138" uid: 41aac4e1-4367-4e17-b4b6-cb7c19151442 spec: {} status: conditions: null createdAt: "2022-08-24T04:44:32Z" gitCommit: 95292c60bb17b77591d6dde5c8636fe6bb4de60d-dirty lastSyncedAt: "2022-08-25T02:50:19Z" managerImage: "longhornio/longhorn-manager:v1.4.0" ownerID: ip-10-0-1-105 state: Ready version: v1.4.0 ``` #### Manager: sync system backups in the backup target ***longhorn-backup-target-controller*** 1. Execute engine binary [system-backup list](#engine-commands). 1. Check for system backups in the backup target that are not in the `SystemBackup` list. 1. Create new `SystemBackup`s and label with `longhorn.io/version: ` for the non-existing system backups. 1. Check for system backups in the `SystemBackup` list that are not in the backup target. 1. Delete `SystemBackup` for the non-existing system backups. 1. Delete `SystemBackup` custom resources when the backup target is empty. ***longhorn-system-backup-controller*** 1. For `SystemBackup` with the `longhorn.io/version: ` label, execute the engine binary [system-backup get-config](#engine-commands) using the ``. 1. Update `SystemBackup` status from the [system backup config](#system-backup-cfg). #### Manager: system backup to the backup target ***POST /v1/systembackups*** ```golang type SystemBackupInput struct { Name string `json:"name"` } ``` 1. Create `SystemBackup`. ```yaml apiVersion: longhorn.io/v1beta2 kind: SystemBackup metadata: name: namespace: longhorn-system ``` 1. Return system backup resource. ```golang type SystemBackup struct { client.Resource Name string `json:"name"` Version string `json:"version,omitempty"` ManagerImage string `json:"managerImage,omitempty"` State longhorn.SystemBackupState `json:"state,omitempty"` CreatedAt string `json:"createdAt,omitempty"` Error string `json:"error,omitempty"` } ``` ***webhook validator*** 1. Skip validation for `SystemBackup` created by the backup target controller. 1. Allow `SystemBackup` to create if met conditions. - The backup target is set. - System backup does not exist in the backup target. ***longhorn-system-backup-controller*** ```none system-backup.zip + metadata.yaml | + yamls/ + apiextensions/ | + customresourcedefinitions.yaml | + kubernetes/ | + clusterrolebindings.yaml | + clusterroles.yaml | + configmaps.yaml | + daemonsets.yaml | + deployments.yaml | + persistentvolumeclaims.yaml | + persistentvolumes.yaml | + podsecuritypolicies.yaml | + rolebindings.yaml | + roles.yaml | + serviceaccounts.yaml | + services.yaml | + longhorn/ + engineimages.yaml + recurringjobs.yaml + settings.yaml + volumes.yaml + backingimages.yaml ``` 1. Create metadata file. ```golang type SystemBackupMeta struct { LonghornVersion string `json:"longhornVersion"` LonghornGitCommit string `json:"longhornGitCommit"` KubernetesVersion string `json:"kubernetesVersion"` LonghornNamespaceUUID string `json:"longhornNamspaceUUID"` SystemBackupCreatedAt metav1.Time `json:"systemBackupCreatedAt"` ManagerImage string `json:"managerImage"` } ``` 1. Backup the Volumes to the backup target. 1. Backup the BackingImages to the backup target after the Volumes. We only backup those BackingImages not being used in this stage because we already backup the BackingImages used by the Volumes when backing up the Volumes automatically. 1. Generated the resource YAML files: 1. Generate the API extension resource YAML file. - CustomResoureDefinitions with API group `longhorn.io`. 1. Generate the Kubernetes resources YAML files. - ServiceAccounts in the Longhorn namespace. - ClusterRoleBinding with any Longhorn ServiceAccounts in the `subjects`. - ClusterRoles with any Longhorn ClusterRoleBindings in the `roleRef`. - Roles in Longhorn namespace. - PodSecurityPolicies with Longhorn Role in the `rules`. - RoleBindings in the Longhorn namespace. - DaemonSets in Longhorn namespace. - Deployments in Longhorn namespace. - `longhorn-storageclass` ConfigMap. - Services in Longhorn namespace. The ClusterIP and ClusterIPs will get removed before converting to the YAML. - StorageClasses with provisioner `driver.longhorn.io`. - PersistentVolumes with Longhorn StorageClass in spec. - PersistentVolumeClaims with Longhorn StorageClass in spec. 1. Generate the Longhorn resources YAML files. - Longhorn Settings. - Longhorn EngineImages. - Longhorn Volumes. - Longhorn RecurringJobs. - Longhorn BackingImages. 1. Archive the files to a zip file. 1. Execute engine binary [system-backup upload](#engine-commands) to upload to the backup target `backupstore/system-backups//`. #### Manager: list system backups (GET /v1/systembackups) 1. List `SystemBackup`. 1. Return collection of [SystemBackup](#system-backup-resource) resource. #### Manager: delete system backup in the backup target ***DELETE /v1/systembackups/{name}*** 1. Deletes `SystemBackup`. ***longhorn-system-backup-controller*** 1. Execute engine binary [system-backup delete](#engine-commands) to remove the system backup in the backup target. 1. Cleanup local generated system backup files and directory that has not been uploaded to the backup target. #### Manager: SystemRestore custom resource ```yaml apiVersion: longhorn.io/v1beta2 kind: SystemRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"longhorn.io/v1beta2","kind":"SystemRestore","metadata":{"annotations":{},"name":"demo-2-restore","namespace":"longhorn-system"},"spec":{"systemBackup":"demo-2"}} creationTimestamp: "2022-08-24T04:44:51Z" finalizers: - longhorn.io generation: 1 name: demo-2-restore namespace: longhorn-system resourceVersion: "283819" uid: ef93355d-3b73-4fdd-bed3-ec5016a6784d spec: systemBackup: demo-2 status: conditions: - lastProbeTime: "" lastTransitionTime: "2022-08-24T04:44:59Z" message: sample error message reason: Download status: "True" type: Error ownerID: ip-10-0-1-113 sourceURL: s3://backupbucket@us-east-1/backupstore/system-backups/v1.4.0/demo-2 state: Error ``` #### Manager: restore system backup from the backup target ***POST /v1/systemrestores*** ```golang type SystemRestoreInput struct { Name string `json:"name"` Version string `json:"version"` SystemBackup string `json:"systemBackup"` } ``` 1. Create `SystemRestore`. ***webhook validator*** 1. Allow `SystemRestore` to create if met conditions. - All volumes are detached. - No other system restore is in progress. - The SystemBackup used in the SystemRestore `Spec.SystemBackup` must exist. ***longhorn-system-restore-controller*** 1. Get system backup config from the backup target. 1. Create a system restore job with the manager image from the [system backup config](#system-backup-cfg). ```yaml apiVersion: batch/v1 kind: Job metadata: name: namespace: longhorn-system spec: backoffLimit: 3 template: metadata: name: spec: containers: - command: - longhorn-manager - system-restore - env: - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: NODE_NAME value: image: imagePullPolicy: IfNotPresent name: volumeMounts: - mountPath: /var/lib/longhorn/engine-binaries/ name: engine nodeSelector: kubernetes.io/hostname: restartPolicy: OnFailure serviceAccount: serviceAccountName: volumes: - hostPath: path: /var/lib/longhorn/engine-binaries/ type: "" name: engine ``` ***command: system-restore*** 1. Start and run longhorn-system-rollout controller. ***longhorn-system-rollout-controller*** 1. Get system backup config from the backup target. 1. Check and create the engine image in the [system backup config](#system-backup-cfg) if the engine image is not in the cluster. 1. Execute engine binary [system-backup download](#engine-commands). 1. Unzip system backup. 1. Decode the resources from files. ```golang type SystemBackupLists struct { customResourceDefinitionList *apiextensionsv1.CustomResourceDefinitionList clusterRoleList *rbacv1.ClusterRoleList clusterRoleBindingList *rbacv1.ClusterRoleBindingList roleList *rbacv1.RoleList roleBindingList *rbacv1.RoleBindingList daemonSetList *appsv1.DaemonSetList deploymentList *appsv1.DeploymentList configMapList *corev1.ConfigMapList persistentVolumeList *corev1.PersistentVolumeList persistentVolumeClaimList *corev1.PersistentVolumeClaimList serviceAccountList *corev1.ServiceAccountList podSecurityPolicyList *policyv1beta1.PodSecurityPolicyList engineImageList *longhorn.EngineImageList recurringJobList *longhorn.RecurringJobList settingList *longhorn.SettingList volumeList *longhorn.VolumeList } ``` - Kubernetes resources from files in the `kubernetes` directory. - API extension resources from files in the `apiextensions` directory. - Longhorn resources from files in the `longhorn` directory. 1. Restore Setting resources and annotate with `longhorn.io/last-system-restore`, and `longhorn.io/last-system-restore-at`. 1. Restore resources asynchronously and annotate with `longhorn.io/last-system-restore`, and `longhorn.io/last-system-restore-at`. - ServiceAccounts. - ClusterRoles. - ClusterRoleBindings. - CustomResourceDefinitions. > **Note:** The controller will not replace the custom resource definitions for version compatibility purposes. Instead, it will add to the existing one if the custom resource definition version is different. Or create if missing. - PodSecurityPolicies. - Roles. - RoleBindings. - ConfigMaps. - Deployments. - DaemonSets. - EngineImages. - BackingImages - StorageClasses. - PersistentVolumes. - PersistentVolumeClaims. - RecurringJobs. 1. Restore Volumes after all resources are restored because Volumes with BackingImages needs to wait until BackingImages are restored. Annotate with `longhorn.io/last-system-restore-backup` if the volume is restored from the backup. 1. Update [SystemRestore](#manager-systemrestore-custom-resource) status and error. 1. Shutdown longhorn-system-rollout controller. #### Engine: commands - [BackupStore: `system-backup upload`](#cmd-system-backup-upload) - [BackupStore: `system-backup delete`](#cmd-system-backup-delete) - [BackupStore: `system-backup download`](#cmd-system-backup-download) - [BackupStore: `system-backup list`](#cmd-system-backup-list) - [BackupStore: `system-backup get-config`](#cmd-system-backup-get-config) #### BackupStore: commands ***Command: system-backup upload*** | Argument | Usage | | -----------------| --------------------------------- | | 0 | the source local file path | | 1 | the destination system backup URL | | Flag | Usage | | -------------- | ---------------------------------------------------- | | git-commit | Longhorn manager git commit of the current cluster | | manager-image | Longhorn Manager image of the current cluster | | engine-image | Longhorn default Engine image of the current cluster | 1. Upload local file to the object store `backupstore/system-backups///system-backup.zip`. 1. Create system backup config. ```golang type SystemBackupConfig struct { Name string Version string GitCommit string BackupTargetURL string ManagerImage string EngineImage string CreatedAt time.Time Checksum string // sha512 } ``` 1. Upload system backup config to the object store `backupstore/system-backups///system-backup.cfg`. ```json { "Name":"demo-2", "Version":"v1.4.0", "GitCommit":"95292c60bb17b77591d6dde5c8636fe6bb4de60d-dirty", "BackupTargetURL":"s3://backupbucket@us-east-1/", "ManagerImage":"c3y1huang/research:000-lh-manager", "EngineImage":"c3y1huang/research:000-lh-ei", "CreatedAt":"2022-08-24T04:44:32.463197176Z", "Checksum":"343b328f97f3ee7af6627eed0d9f42662633c0a2348d4eddaa8929a824452fdde0de6f5620c3ea309579bb58381e48bbb013e92492924fcd3dc57006147e2626" } ``` ***Command: system-backup download*** | Argument | Usage | | -----------------| ----------------------------- | | 0 | the source system backup URL | | 1 | the destination local path | 1. Download the system backup zip file from object store to the local path. 1. Verify the checksum of the system backup zip file. Delete the downloaded file when the checksum is mismatched. ***Command: system-backup delete*** | Argument | Usage | | -----------------| ----------------------- | | 0 | the system backup URL | 1. Delete a system backup in the object store. ***Command: system-backup list*** | Argument | Usage | | -----------------| -------------------------------------------------- | | 0 | the backup target URL where system backup exists | 1. List system backups in the object store. ``` map[string]string{ "demo-1": "backupstore/system-backups/v1.4.0/demo-1", "demo-2": "backupstore/system-backups/v1.4.0/demo-2", } ``` ***Command: system-backup get-config*** | Argument | Usage | | -----------------| ----------------------- | | 0 | the system backup URL | 1. Output the [system backup config](#system-backup-cfg) from the object store. ### Test plan #### System Backup - Test system backup to the backup target. - Test system backup should fail when the backup target is empty. - Test system backup should fail when the backup target is unreachable. #### System Restore ***Same Version*** - Test system restore to the same cluster. - Test system restore to a new cluster. - Test system restore can restore volume data from the last backup. - Test system restore should fail when the volume is attached. - Test system restore when another one is in progress. - Test system restore sync from the backup target. - Test system restore each resource when exist in the cluster. - Test system restore each resource when not exist in the cluster. - Test system restore failed to unzip. - Test system restore failed to restore resources. ***Version Jump*** - Test system restore to lower Longhorn version of each Longhorn installation method (kubectl/helm/Rancher). - Test system restore to higher Longhorn version of each Longhorn installation method (kubectl/helm/Rancher). - Test system restore to cluster with multiple engine images. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20220922-snapshot-checksum-and-bit-rot-detection.md ================================================ # Snapshot Checksum Calculation and Bit Rot Detection ## Summary Longhorn system supports volume snapshotting and stores the snapshot disk files on the local disk. However, it is impossible to check the data integrity of snapshots due to the lack of the checksums of the snapshots in current implementation. As a result, if the underlying storage bit rots, no way is available to detect the data corruption and repair the replicas. In the enhancement, the snapshot checksum is calculated after the snapshot is taken and is checked periodically. When a corrupted snapshot is detected, a replica rebuild is triggered to repair the snapshot. ### Related Issues - [[IMPROVEMENT] Introduce checksum for snapshots](https://github.com/longhorn/longhorn/issues/4210) - [[FEATURE] automatic identifying of corrupted replica (bit rot detection)](https://github.com/longhorn/longhorn/issues/3198) ## Motivation ### Goals - Automatic snapshot hashing - Identify corrupted snapshot - Trigger replica rebuild when a corrupted snapshot is detected ### Non-goals - The hashing/checking mechanism is applied to detached volumes - Support concurrent snapshot hashing - In current architecture, the instance-manager-r does not have a proxy, so the snapshot requests are directly sent to the replica processes’ sync-agent servers. Hence, the concurrent limit cannot achieved in the instance-manager-r internally. - From the benchmarking result, the checksum calculation eats too much io resource and impacts the system performance a lot. We also don’t know if the longhorn disks on a same physical disk or not. If they are on the same physical disk and the concurrent limit is larger than 1, the other workloads will be impacted significantly, and there might be a disaster for the entire system. ## Proposal ### User Stories Bit rot in storage is rare but real, and it can corrupt the data silently. Longhorn supports volume snapshotting and restoring a volume to a previous version. However, due to the lack of the checksums of the snapshots in current implementation, it is impossible to ensure the data integrity of the replicas/snapshots. Although, we provide a method ([ref](https://longhorn.io/docs/1.3.1/advanced-resources/data-recovery/corrupted-replica/)) to identify the corrupted snapshots/replicas, the process is tedious and time-consuming for users. ### User Experience In Detail 1. Users' operations will not be affected by snapshot hashing and checking. 2. The system will consume computing and disk IO resources while hashing snapshot disk files. In the meantime, the CPU usages are 380m and 900m when computing the CRC64 (ISO) and SHA256 values, respectively. In the implementation, the CRC64 (ISO) is utilized for detecting corruption. - The snapshot hashing benchmarking result is provided - The read performance will be impacted as well, as summarized in the below table. - Environment - Host: AWS EC2 c5d.2xlarge - CPU: Intel(R) Xeon(R) Platinum 8124M CPU @ 3.00GHz - Memory: 16 GB - Network: Up to 10Gbps - Kubernetes: v1.24.4+rke2r1 - Result - Disk: 200 GiB NVMe SSD as the instance store - 100 GiB snapshot with full random data ![Snapshot Hash Performance Impact (SSD)](image/snapshot_hash_ssd_perf.png) - Disk: 200 GiB throughput optimized HDD (st1) - 30 GiB snapshot with full random data ![Snapshot Hash Performance Impact (HDD)](image/snapshot_hash_hdd_perf.png) #### CLI Add `snapshot hash` and `snapshot hash-status` commands - `snapshot hash` issues a snapshot hashing request to engine. - Usage: `longhorn --url ${engine-ip}:${engine-port} snapshot hash tcp://${replica-sync-agent-ip}:${replica-sync-agent-port} --snapshot-name ${name}` - `snapshot hash-status` requests the snapshot hashing status from engine. - Usage: `longhorn --url ${engine-ip}:${engine-port} snapshot hash-status tcp://${replica-sync-agent-ip}:${replica-sync-agent-port}` - `snapshot hash-cancel` cancels the snapshot hashing task. - Usage: `longhorn --url ${engine-ip}:${engine-port} snapshot hash-cancel tcp://${replica-sync-agent-ip}:${replica-sync-agent-port}` #### Engine Proxy gRPC API Add `SnapshotHash`, `SnapshotHashStatus` and `SnapshotHashCancel` methods and their request and response messages. - `SnapshotHash` issues a snapshot hashing request to engine. - `SnapshotHashStatus` requests the snapshot hashing status from engine. - `SnapshotHashCancel` cancels the snapshot hashing task. #### Replica Sync-Agent gRPC API Add `SnapshotHash`, `SnapshotHashStatus` and `SnapshotHashCancel` methods and their request and response messages. - `SnapshotHash` issues a snapshot hashing request to replica sync-agent. - `SnapshotHashStatus` requests the snapshot hashing status from replica sync-agent. - `SnapshotHashCancel` cancels the snapshot hashing task. ## Design ### Implementation Overview #### Global Settings - **snapshot-data-integrity** - Description: A global setting for enabling or disabling snapshot data integrity checking mode. - Type: string - Value: - disabled: Disable snapshot disk file hashing and data integrity checking. - enabled: Enables periodic snapshot disk file hashing and data integrity checking. To detect the filesystem-unaware corruption caused by bit rot or other issues in snapshot disk files, Longhorn system periodically hashes files and finds corrupted ones. Hence, the system performance will be impacted during the periodical checking. - fast-check: Enable snapshot disk file hashing and fast data integrity checking. Longhorn system only hashes snapshot disk files if they are not hashed or if the modification time changed. In this mode, filesystem-unaware corruption cannot be detected, but the impact on system performance can be minimized. - Default: `disabled` - **snapshot-data-integrity-immediate-checking-after-snapshot-creation** - Description: Hashing snapshot disk files impacts the performance of the system. The immediate snapshot hashing and checking can be disabled to minimize the impact after creating a snapshot. - Type: bool - Default: `false` - **snapshot-data-integrity-cron-job** - Description: The setting is a set of five fields in a line, indicating when Longhorn checks the data integrity of snapshot disk files. - Type: string (Cron job format) - Default: `0 0 */7 * *` (once a week) #### CRDs - Volume - Add `volume.spec.snapshotDataIntegrity` for setting the volume's snapshot data integrity checking mode. The value can be `ignored`, `disabled`, `enabled` or `fast-check`. - `ignored` means the the volume's snapshot check is following the global setting `snapshot-data-integrity`. - After upgrading Longhorn-system, the value is set to `ignored` for an existing volumes whose `volume.spec.snapshotDataIntegrity` is not set. - For a newly created volume, the value is `ignored` by default. - Snapshot - Add `snapshot.status.checksum` for recording the snapshot `crc64(iso)` checksum. - Node - Add `node.status.snapshotPeriodicCheckStatus.state` for indicating current periodic check state. The value can be `idle` or `in-progress`. - Add `node.status.snapshotPeriodicCheckStatus.lastCheckedAt` for recording the start timestamp of the last checksum checking. #### Automatic Snapshot Checksum Hashing and Checking ![Snapshot Checksum Hashing and Checking Flow](image/snapshot_checksum_calculation_flow.png) 1. Node controller creates a snapshot monitor for hashing snapshot disk files as well as checking their data integrity. The monitor is consist of - **1 goroutine `processSnapshotChangeEvent()`**: Send a snapshot hashing/checking task to the `snapshotCheckTaskQueue` workqueue after receiving one snapshot `UPDATE` event. - **1 goroutine `processPeriodicSnapshotCheck()`**: Periodically create snapshot hashing/checking tasks. The period is determined by the global setting `snapshot-data-integrity-cron-job`. When the job is started, it populates engines' snapshots and sends snapshot hashing/checking tasks to the `snapshotCheckTaskQueue` channel. - **N task workers**: Issue hashing requests to engines and detect corrupted replicas according to the results. 2. Task workers fetch tasks from `snapshotCheckTaskQueue` and check if the snapshot disk file needs to be hashed. The rules are - If one of the following conditions are met, do not hash the file - Volume-head disk file, i.e. `Volume Head` in the following figure - System-generated snapshot disk file, e.g. `ccb017f6` and `9a8d5c9c`. ![Snapshot Hash](image/snapshot_hash.png) 3. Issue snapshot hashing requests to their associated engines. Then, the checksum of the snapshot disk file is calculated individually in the replica process. To ensure only one in-progress calculation, the worker holds the per-node file lock (`/host/var/lib/longhorn/.lock/hash`) when calculating the checksum to avoid significant storage performance drop caused by the concurrent calculations. 4. The worker waits until each snapshot disk file's checksum calculation has been completed. It periodically polls the engine and checks the status during the waiting period. 5. The worker gets the result once the calculation is completed. The result is like ``` map[string]string{ "pvc-abc-r-001": 0470c08fbc4dc702, "pvc-abc-r-002": 0470c08fbc4dc702, "pvc-abc-r-003": ce7c12a4d568fddf, } ``` 6. The **final checksum** is determined by the majority of the checksums with `SilentCorrupted=false` from replicas. For instance, the **final checksum** of the result in 4. is `0470c08fbc4dc702`. - When all checksums differ, the **final checksum** is unable to be determined - If `snapshot.status.checksum` is empty - Set all replicas to `ERR` - If `snapshot.status.checksum` is already set - Use the `snapshot.status.checksum` as the **final checksum**, and set the replicas that have mismatching checksums to `ERR` - When the **final checksum** is successfully determined - Assign the **final checksum** to `snapshot.status.checksum` - Set the replica to `ERR` if its snapshot disk file's checksum is not equal to `snapshot.status.checksum` - If the **final checksum** cannot be determined, the event of the corruption detected is also emitted. - For example, Longhorn will not do any error handling and just emits a event when the silent corruption is found in a single-replica volume. 7. Then, the replicas in `ERR` mode will be rebuilt and fixed. The event of the corruption detected is also emitted. #### Snapshot Disk File Hashing in Replica Process When the replica process received the request of snapshot disk file hashing, the checking mode is determined by `volume.spec.snapshotDataIntegrity`. If the value is `ignored`, the checking mode follows the global setting `snapshot-data-integrity`. - **`fask-check`** - Flow 1. Get `ctime` information of the snapshot disk file. 2. Get the value of the extended attribute `user.longhorn-system.metadata` recording the checksum and `ctime` of the file in the last calculation. The value of `user.longhorn-system.metadata` is JSON formatted string and records `hashing method`, `checksum`, `ctime` and etc. 3. Compare the `ctime` from 1. and 2. Recalculate the checksum if one of the conditions is met - The two `ctime` are mismatched. - 2's `ctime` is not existing. 4. Ensure that the checksum is reliable by getting the `ctime` information of the disk file again after the checksum calculation. - If it is matched with 1's `ctime`, update the extended attribute with the latest result. - Instead, it indicates the file is changed by snapshot pruning, merging or other operations. Thus, recalculate the checksum. A maximum retries controls the recalculation. 5. Return checksum or error to engine. - **enabled** - Because the silent data corruption in snapshot disk files can be caused by the host's storage device such as bit rot or somewhere within the storage stack. Filesystem cannot be aware of the corruption. To detect the corruption, the checksums of the disk files are always be recalculated and return back to engine. Silent corruption is detected when the disk file's `ctime` matches the `ctime` in the extended attribute, but the checksums do not match. The extended attribute will not be updated, and the `SilentCorrupted` of the hash status will be set to `true`. - **disabled** - Do nothing. ### Test Plan **Integration tests** - Test snapshot disk files hashing - Compare the checksum recorded in `snapshot.status.checksum` and the checksum (calculated by a [3rd-party CRC64 checksum utility](#3rd-party-crc64-checksum-utility-for-test-plan) of each replica's snapshot disk file. - Test snapshot disk files check - Corrupt a snapshot disk file in one of the replicas. Then, check the corruption is detected by Longhorn, and the replica rebuilding should be triggered. ## Note[optional] ### 3rd-party CRC64 Checksum Utility for Test Plan - Install Java (`apt install default-jre default-jdk`) - Download jacksum (`wget https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar`) - Calculate checksum by `java -jar jacksum-3.4.0.jar -a crc64_go-iso ${path-to-file}` ================================================ FILE: enhancements/20221018-record-recurring-jobs-in-the-backup-volume.md ================================================ # Record Recurring Jobs in the Backup Volume ## Summary Provide a way that users can record recurring jobs/groups during the backup creation and restore them during the backup restoration. The feature will back up all recurring jobs/groups of the volume into the backup volume configuration on the backup target and restore all jobs when users want to create a volume from a backup with recurring jobs/groups stored in the backup volume. ### Related Issues https://github.com/longhorn/longhorn/issues/2227 ## Motivation ### Goals 1. Backup or update all recurring jobs/groups to the backup volume during the backup creation. 2. Create recurring jobs/groups and bind them to the volume restored from a backup optionally. 3. It is backward compatible for current backups w/o recurring jobs info. ### Non-goals [optional] 1. Not support to back up or restore a specific recurring job/group. 2. A DR volume will not restore recurring jobs/groups during a backup restoration. ## Proposal 1. Add a global boolean setting `restore-volume-recurring-jobs`. Default value is `false`. When users create a volume from a backup and this setting is set to be `true`, it will automatically restore all recurring jobs/groups stored in the backup volume. 2. Add a customized string parameter `RestoreVolumeRecurringJob` in `Volume` CR. Default value is `"ignored"`. `"enabled"` is to restore recurring jobs/groups. By contrast, `"disabled"` is not to restore. Users can override the default behavior during the restoration at runtime by this parameter. ### User Stories #### Story 1 Users can simply create recurring jobs from restoring a backup created by other Longhorn systems. And continue to back up this restoring volume to the backup target with the same recurring jobs settings. #### Story 2 When the users delete recurring jobs of the volume by accident, they could restore some recurring jobs from the backup volume by restoring a backup if they do not want to create recurring jobs manually. ### User Experience In Detail #### Via Longhorn GUI - Users can set `restore-volume-recurring-jobs` to be `true` on the `Settings` page. - When users restore a backup to create a volume, they can see the recurring jobs/groups are restored and enabled automatically on the volume details page. - Users can check the checkbox `enabled` or `disabled` to override the global setting of restoring recurring jobs/groups. #### Via `kubectl` - User can use the command `kubectl -n longhorn-system edit settings` to set `restore-volume-recurring-jobs` to be `true` - Users can set `Volume.spec.restoreVolumeRecurringJob` to `enabled` or `disabled` to override the global setting of restoring recurring jobs/groups when creating a volume from a backup. - When users create a volume by restoring a backup, they can see the recurring jobs/groups are restored as `RecurringJob` CRs and labeled in the `Volume` CR. ```yaml ... kind: Volume metadata: labels: longhornvolume: restore-demo name: restore-demo namespace: longhorn-system spec: RestoreVolumeRecurringJob: "enabled" fromBackup: "nfs://nfs-sever.com:/opt/shared-path/?backup=backup-f6d9b9caa9444543&volume=backup1" ... ``` ### API changes Add a string parameter `RestoreVolumeRecurringJob` to the `Volume` struct utilized by the http client, This ends up being stored in `Volume.spec.restoreVolumeRecurringJob` of the volume CR. ## Design ### Implementation Overview 1. Add a global boolean setting `restore-volume-recurring-jobs`. Default value is `false`. It will restore all recurring jobs/groups of the backup volume during a backup restoration if this setting is set to be `true`. 2. Add the parameter `RestoreVolumeRecurringJob` into `Volume` struct of api/model.go and volume CR. Default value is `"ignored"`. 3. Store all recurring jobs information of the volume into the backup volume configuration on the backup target during the backup creation. - We had saved the `"RecurringJob":"c-jaim49"` information in the `spec.labels` of the backup CR to show you the backup is created by a recurring job and this information will also be stored into backup volume configuration on the backup target and update to `status.labels` of the backup volume CR but it only contains the recurring job name and it will be changed after any recurring job creates a backup. - Now we back up the details of recurring jobs/groups information into backup volume configuration on the backup target and synchronized to `status.labels` of the backup volume CR. When users need to restore recurring jobs/groups to the current Longhorn system or another, it will get the recurring jobs/groups configuration from backup volume CR. ```text Backup Controller queue ┌───────────────┐ ┌───────────────────────┐ ┌┐ ┌┐ ┌┐ │ │ │ │ ... ││ ││ ││ ──────► │ ... | ──────► │ reconcile() │ └┘ └┘ └┘ │ │ │ │ └───────────────┘ └──────────┬────────────┘ │ instance-manager ┌──────────▼────────────┐ ┌──────────────────────┐ ┌──────────────────────┐ │ │ │ │ │ │ │ enableBackupMonitor() │ ──────► │ NewBackupMonitor() │ ... ─► │ SnapshotBackup() │ ... │ │ │ │ │ │ └───────────────────────┘ └──────────────────────┘ └──────────────────────┘ ``` 1. The `backup_controller` will be responsible for collecting recurring jobs information and send it to the backup monitor when detecting a new backup CR created. 2. The `backup_monitor` will put recurring jobs information with a new key `VolumeRecurringJobs` into the `spec.labels` of the backup CR and trigger the backup creation. 3. Recurring jobs information in the labels will be stored into the backup volume configuration by `backupstore`. Example of recurring jobs/groups information stored in the backup volume configuration. ```json { ..., "Labels": { "RecurringJob":"c-jaim49", "VolumeRecurringJobInfo": "{ \"c-jaim49\": { \"jobSpec\": {\"name\":\"c-jaim49\",\"task\":\"backup\",\"cron\":\"0/1 * * * *\",\"retain\":3,\"concurrency\":1}, \"fromGroup\":null, \"fromJob\":true }, \"c-qakbzx\": { \"jobSpec\":{\"name\":\"c-qakbzx\",\"groups\":[\"default\"],\"task\":\"backup\",\"cron\":\"0 0 * * *\",\"retain\":5,\"concurrency\":3}, \"fromGroup\":[\"default\"], \"fromJob\":false }, \"c-ua7pxz\": { \"jobSpec\":{\"name\":\"c-ua7pxz\",\"groups\":[\"testgroup01\"],\"task\":\"backup\",\"cron\":\"0/10 0/2 * * *\",\"retain\":3,\"concurrency\":3}, \"fromGroup\":[\"testgroup01\"], \"fromJob\":true } }", "longhorn.io/volume-access-mode":"rwo" }, ..., } ``` 4. Create all recurring jobs if they do not exist when restoring a backup with the setting `restore-volume-recurring-jobs` being `true` or `Volume.spec.restoreVolumeRecurringJob` being `"enabled"`. ```text Volume Controller queue ┌───────────────┐ ┌───────────────────────┐ ┌┐ ┌┐ ┌┐ │ │ │ │ ... ││ ││ ││ ──────► │ ... | ──────► │ syncVolume() │ └┘ └┘ └┘ │ │ │ │ └───────────────┘ └──────────┬────────────┘ │ ┌───────────▼─────────────┐ │ │ │ updateRecurringJobs() │ │ │ └─────────────────────────┘ ``` 1. Create all recurring jobs gotten from the backup volume CR if they do not exist or configuration is different and set volume labels of recurring jobs to be `"enabled"` before a restoration starts. ### Test plan #### Prepare 1. Create a volume and attach it to a node or a workload. 2. Create some recurring jobs (some are in groups) 3. Label the volume with created recurring jobs (some are in groups) 4. Create a backup or wait for a recurring job starting 5. Wait for backup creation completed. 6. Check if recurring jobs/groups information is stored in the backup volume configuration on the backup target #### Recurring Jobs exist 1. Create a volume from the backup just created. 2. Check the volume if it has labels of recurring jobs and groups. #### Recurring Jobs do not exist 1. Delete recurring jobs that are already stored in the backup volume on the backup. 2. Create a volume from the backup just created. 3. Check if recurring jobs have been created. 4. Check if restoring volume has labels of recurring jobs and groups. ### Upgrade strategy This enhancement doesn't require an upgrade strategy. ================================================ FILE: enhancements/20221024-longhorn-volumeattachment.md ================================================ # Consolidate Volume Attach/Detach Implementation ## Summary There are several cases related to (auto) volume attach/detach, right now we leverage the volume attributes to achieve that, but it's better to introduce a new resource (longhorn volumeattachement) to have complete context for each scenario. ### Related Issues https://github.com/longhorn/longhorn/issues/3715 ## Motivation ### Goals Introduce a new resource (Longhorn volumeattachement) to cover the following scenarios for Longhorn volume's AD: 1. Traditional CSI attachment (pod -> csi-attacher -> Longhorn API) 1. Traditional UI attachment (Longhorn UI -> Longhorn API) 1. Auto attach/detach volume for K8s CSI snapshot 1. Auto attach/detach volume for recurring jobs 1. Auto attach/detach volume for volume cloning 1. Auto attach/detach volume for auto salvage feature 1. Refactor RWX mechanism's volume attachment/detachment (in share manager lifecycle) 1. Volume live migration 1. Consider how to upgrade from previous Longhorn version which doesn't have VA resource yet ### Non-goals [optional] NA ## Proposal This is where we get down to the nitty-gritty of what the proposal actually is. ### User Stories #### Story 1 Before this feature there are race conditions between Longhorn auto-reattachment logic and CSI volume attachment that sometimes result in the volume CR in a weird state that volume controller can nerve resolve. Ref https://github.com/longhorn/longhorn/issues/2527#issuecomment-966597537 After this feature, the race condition should not exist #### Story 2 Before this feature, the user cannot take a CSI snapshot for detached Longhorn volume. After this feature, user should be able to do so Ref: https://github.com/longhorn/longhorn/issues/3726 #### Story 3 Make the attaching/detaching more resilient and transparent. User will see clearly who is requesting the volume to be attached in the AD ticket. Also, volume controller will be able to reconcile the volume in any combination value of (volume.Spec.NodeID, volume.Status.CurrentNodeID, and volume.Status.State). See the [volume-controller-ad-logic](./assets/images/longhorn-volumeattachment/volume-controller-ad-logic.png) ## Design & Implementation Overview 1. Create a new CRD called VolumeAttachment with the following structure: ```go type AttachmentTicket struct { // The unique ID of this attachment. Used to differentiate different attachments of the same volume. // +optional ID string `json:"id"` // +optional Type AttacherType `json:"type"` // The node that this attachment is requesting // +optional NodeID string `json:"nodeID"` // Optional additional parameter for this attachment // +optional Parameters map[string]string `json:"parameters"` // A sequence number representing a specific generation of the desired state. // Populated by the system. Read-only. // +optional Generation int64 `json:"generation"` } type AttachmentTicketStatus struct { // The unique ID of this attachment. Used to differentiate different attachments of the same volume. // +optional ID string `json:"id"` // Indicate whether this attachment ticket has been satisfied Satisfied bool `json:"satisfied"` // Record any error when trying to fulfill this attachment // +nullable Conditions []Condition `json:"conditions"` // A sequence number representing a specific generation of the desired state. // Populated by the system. Read-only. // +optional Generation int64 `json:"generation"` } type AttacherType string const ( AttacherTypeCSIAttacher = AttacherType("csi-attacher") AttacherTypeLonghornAPI = AttacherType("longhorn-api") AttacherTypeSnapshotController = AttacherType("snapshot-controller") AttacherTypeBackupController = AttacherType("backup-controller") AttacherTypeVolumeCloneController = AttacherType("volume-clone-controller") AttacherTypeSalvageController = AttacherType("salvage-controller") AttacherTypeShareManagerController = AttacherType("share-manager-controller") AttacherTypeVolumeRestoreController = AttacherType("volume-restore-controller") AttacherTypeVolumeEvictionController = AttacherType("volume-eviction-controller") AttacherTypeVolumeExpansionController = AttacherType("volume-expansion-controller") AttacherTypeBackingImageDataSourceController = AttacherType("bim-ds-controller") AttacherTypeVolumeRebuildingController = AttacherType("volume-rebuilding-controller") ) const ( AttacherPriorityLevelVolumeRestoreController = 2000 AttacherPriorityLevelVolumeExpansionController = 2000 AttacherPriorityLevelLonghornAPI = 1000 AttacherPriorityLevelCSIAttacher = 900 AttacherPriorityLevelSalvageController = 900 AttacherPriorityLevelShareManagerController = 900 AttacherPriorityLevelSnapshotController = 800 AttacherPriorityLevelBackupController = 800 AttacherPriorityLevelVolumeCloneController = 800 AttacherPriorityLevelVolumeEvictionController = 800 AttacherPriorityLevelBackingImageDataSourceController = 800 AttachedPriorityLevelVolumeRebuildingController = 800 ) const ( TrueValue = "true" FalseValue = "false" AnyValue = "any" AttachmentParameterDisableFrontend = "disableFrontend" AttachmentParameterLastAttachedBy = "lastAttachedBy" ) const ( AttachmentStatusConditionTypeSatisfied = "Satisfied" AttachmentStatusConditionReasonAttachedWithIncompatibleParameters = "AttachedWithIncompatibleParameters" ) // VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment type VolumeAttachmentSpec struct { // +optional AttachmentTickets map[string]*AttachmentTicket `json:"attachmentTickets"` // The name of Longhorn volume of this VolumeAttachment Volume string `json:"volume"` } // VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment type VolumeAttachmentStatus struct { // +optional AttachmentTicketStatuses map[string]*AttachmentTicketStatus `json:"attachmentTicketStatuses"` } // VolumeAttachment stores attachment information of a Longhorn volume type VolumeAttachment struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Spec VolumeAttachmentSpec `json:"spec,omitempty"` Status VolumeAttachmentStatus `json:"status,omitempty"` } ``` 1. Modify volume controller 1. Repurpose the field `volume.Status.CurrentNodeID` so that `volume.Status.CurrentNodeID` is only set once we are fully attached and is only unset once we are fully detached. See this state flow for full detail: [volume-controller-ad-logic](./assets/images/longhorn-volumeattachment/volume-controller-ad-logic.png) 2. Deprecate `volume.Status.PendingNodeID` and the auto-salvage logic. We will have a dedicated `salvage-controller` as describe in the below section. 1. Create a controller, VolumeAttachment controller (AD controller). This controller watches the VolumeAttachment objects of a volume. 1. When AD controller sees a newly created ticket in `VolumeAttachment.Spec.AttachmentTickets` object. 1. If `volume.Spec.NodeID` is non-empty 1. Do nothing. It will wait for the volume to be fully detached first before setting `volume.Spec.NodeID` 1. If `volume.Spec.NodeID` is empty 1. Wait for `volume.Status.State` to be `detached` 1. Then select an attachment ticket from `va.Spec.AttachmentTickets` based on priority level of the tickets. If 2 ticket has same priority, select the ticket with shorter name. 2. Set the `vol.Spec.NodeID = attachmentTicket.NodeID` to attach the volume 1. When AD controller check the list of tickets in `VolumeAttachment.Spec.AttachmentTickets`. If no ticket is requesting the `volume.Spec.NodeID`, Ad controller set `volume.Spec.NodeID` to empty 1. AD controller watch volume CR and set ticket status accordingly in the `va.Status.AttachmentTicketStatuses` 1. If the VolumeAttachment object is pending deletion, There is no special resource need to be cleanup, directly remove the finalizer for the VolumeAttachment > Note that the priority of ticket is determine in the order: volume data restoring > user workload > snapshot/backup operations 1. Traditional CSI attachment (pod -> csi-attacher -> Longhorn API) 1. csi-attacher send attaching request to longhorn-csi-plugin 1. longhorn-csi-plugin sends attaching request to longhorn-manager with pod-id and attacherType `csi-attacher` 1. longhorn manager create a VolumeAttachment object with this spec: ```yaml metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: csi-attacher: : id: type: "csi-attacher" volume: nodeID: status: attached: false ``` 1. longhorn-csi-plugin watch the `volumeAttachment.Status.Attached` and `volumeAttachment.Status.AttachError` and return corresponding respond to csi-attacher 1. Traditional UI attachment (Longhorn UI -> Longhorn API) 1. Longhorn UI send attaching request to longhorn-manager with attacherType `longhorn-api` 1. longhorn manager create a VolumeAttachment object with this spec: ```yaml metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: longhorn-api: "": id: "" type: "longhorn-api" volume: nodeID: status: attached: false ``` 1. Longhorn UI watches the `volumeAttachment.Status.Attached` and `volumeAttachment.Status.AttachError` and display the correct message 1. Auto attach/detach volume Longhorn snapshot 1. Snapshot controller watches the new Longhorn snapshot CR. If the snapshot CR request a new snapshot, snapshot controller create a new VolumeAttachment object with the content: ```yaml metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: snapshot-controller: : id: type: "snapshot-controller" volume: nodeID: status: attached: false ``` 1. Snapshot controller wait for volume to be attached and take the snapshot 1. Auto attach/detach volume Longhorn backup 1. Backup controller watches the new Longhorn backup CR. If the backup CR request a new backup, backup controller create a new VolumeAttachment object with the content: ```yaml metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: backup-controller: : id: type: "backup-controller" volume: nodeID: status: attached: false ``` 1. Backup controller wait for volume to be attached and take the backup 1. Auto attach/detach volume for K8s CSI snapshot 1. csi-snappshotter send a request to longhorn-csi-plugin to with snapshot name and volume name 1. longhorn-csi-plugin send snapshot request to Longhorn manager 1. Longhorn manager create a snapshot CR 1. longhorn-csi-plugin watches different snapshot CR status and respond to csi-snapshotter 1. Auto attach/detach volume for recurring jobs 1. recurring job deploys backup/snapshot CR 1. recurring job watch the status of backup/snapshot CR for the completion of the operation 1. Auto attach/detach volume for volume cloning 1. Create a new controller: cloning-controller 1. This controller will watch new volume that need to be cloned 1. For a volume that needs to be cloned, cloning controller deploy a VolumeAttachment for both target volume and old volume: ```yaml # target volume metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: cloning-controller: : id: type: "cloning-controller" volume: nodeID: status: attached: false # source volume metadata: finalizers: - longhorn.io labels: longhornvolume: nodeID: spec: attachers: cloning-controller: : id: type: "cloning-controller" volume: nodeID: status: attached: false ``` 1. Cloning controller watch for the cloning status and delete the VolumeAttachment upon completed 1. Auto attach/detach volume for auto salvage feature We create a new controller to check if the volume is faulted and detach the volume for detach. After the volume is auto detach and replica is auto-salvaged by the volume controller, the AD controller will check the VolumeAttachment object and reattach the volume to the correct node With this design, we no longer need the volume.Status.PendingNodeID which was the source of some race conditions 1. Refactor RWX mechanism's volume attachment/detachment (in share manager lifecycle) Each pod that use the RWX volume will directly request a CSI ticket. The share-manager controller watch the csi ticket and create share-manager ticket when there are one or more csi ticket exist. Then AD controller will attach the volume to the node that is being requested by share-manager ticket. AD controller ignore the CSI ticket for RWX volume. ### API changes We are adding new APIs to operate on Snapshot CRD directly ```go "snapshotCRCreate": s.SnapshotCRCreate, "snapshotCRList": s.SnapshotCRList, "snapshotCRGet": s.SnapshotCRGet, "snapshotCRDelete": s.SnapshotCRDelete, ``` ### Test plan Refer to the test plan https://github.com/longhorn/longhorn/issues/3715#issuecomment-1563637861 ### Upgrade strategy In the upgrade path, list all volume and create VolumeAttachment for the them. For the volume that currently being used by CSI workload/Longhorn UI we create CSI ticket/Longhorn UI ticket to keep them being attached ================================================ FILE: enhancements/20221024-pv-encryption.md ================================================ # Add PV encryption support ## Summary This enhancement adds support for user configured (storage class, secrets) encrypted volumes, this in return means that backups of that volume end up also being encrypted. ### Related Issues - [RWO encryption support](https://github.com/longhorn/longhorn/issues/1859) - [Backup encryption](https://github.com/longhorn/longhorn/issues/902) - [RWX encryption support]() // TODO: create issue - [Expansion support](https://github.com/longhorn/longhorn/issues/2794) - [Restore support]() // TODO: create issue - [VolumeMode Block support]() // TODO: create issue ## Motivation ### Goals - user is able to create & use an encrypted volume with cipher customization options - user is able to configure the keys that are used for encryption - user is able to take backups from an encrypted volume - user is able to restore an encrypted backup to a new encrypted volume ### Non-goals - external key management support, currently keys utilize kubernetes secrets - rotating key support, user can do this manually though as a workaround - securing of secrets, user is responsible for cluster setup and security of the secrets ## Proposal ### User Stories All regular longhorn operations should also be supported for encrypted volumes, therefore the only user story that is mentioned is how to create and use an encrypted volume. #### Create and use an encrypted volume - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create the secret for that volume in the configured namespace with customization options of the cipher for instance `cipher`, `key-size` and `hash` - create a pvc that references the created storage class - volume will be created then encrypted during first use - afterwards a regular filesystem that lives on top of the encrypted volume will be exposed to the pod ### User Experience In Detail Creation and usage of an encrypted volume requires 2 things: - the storage class needs to specify `encrypted: "true"` as part of its parameters. - secrets need to be created and reference for the csi operations need to be setup. - see below examples for different types of secret usage. The kubernetes sidecars are responsible for retrieval of the secret and passing it to the csi driver. If the secret hasn't been created the PVC will remain in the Pending State. And the side cars will retry secret retrieval periodically, once it's available the sidecar container will call `Controller::CreateVolume` and pass the secret after which longhorn will create a volume. #### Create storage class that utilizes a global secret (all volumes use the same key) The below storage class uses a global secret named `longhorn-crypto` in the `longhorn-system` namespace. ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-global provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" csi.storage.k8s.io/provisioner-secret-name: "longhorn-crypto" csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-stage-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" ``` The global secret reference by the `longhorn-crypto-global` storage class. This type of setup means that all volumes share the same encryption key. ```yaml apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets CRYPTO_KEY_CIPHER: "aes-xts-plain64" # this is optional CRYPTO_KEY_HASH: "sha256" # this is optional CRYPTO_KEY_SIZE: "256" # this is optional ``` #### Create storage class that utilizes per volume secrets The below storage class uses a per volume secret, the name and namespace of the secret is based on the pvc values. These templates will be resolved by the external sidecars and the resolved values end up as Secret refs on the PV. ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-per-volume provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" csi.storage.k8s.io/provisioner-secret-name: ${pvc.name} csi.storage.k8s.io/provisioner-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-publish-secret-name: ${pvc.name} csi.storage.k8s.io/node-publish-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-stage-secret-name: ${pvc.name} csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} ``` ### API changes add a `Encrypted` boolean to the `Volume` struct utilized by the http client, this ends up being stored in `Volume.Spec.encrypted` of the volume cr. Storing the `Encrypted` value is necessary to support encryption for RWX volumes. ## Design ### Implementation Overview Host requires `dm_crypt` kernel module as well as `cryptsetup` installed. We utilize the below parameters from a secret, - `CRYPTO_KEY_PROVIDER` allows us in the future to add other key management systems - `CRYPTO_KEY_CIPHER` allow users to choose the cipher algorithm when creating an encrypted volume by `cryptsetup` - `CRYPTO_KEY_HASH` specifies the hash used in the LUKS key setup scheme and volume key digest - `CRYPTO_KEY_SIZE` sets the key size in bits. The argument has to be a multiple of 8 and the maximum interactive passphrase length is 512 (characters) ```yaml CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets CRYPTO_KEY_CIPHER: "aes-xts-plain64" # this is optional CRYPTO_KEY_HASH: "sha256" # this is optional CRYPTO_KEY_SIZE: "256" # this is optional ``` - utilize host `dm_crypt` kernel module for device encryption - utilize host installed `cryptsetup` for configuration of the crypto device - add csi driver `NodeStageVolume` support to handle device global per node mounting, we skip mounting for volumes that are being used via `VolumeMode: Block` - refactor csi driver NodePublishVolume to bind mount the `staging_path` into the `target_path` we utilize a bind mount for `VolumeMode: Mount` we do a regular device file creation for `VolumeMode: Block` - during csi `NodeStageVolume` encrypt (first time use) / open regular longhorn device - this exposes a crypto mapped device (/dev/mapper/) - mount crypto device into `staging_path` - during csi `NodeUnstageVolume` unmount `staging_path` close crypto device ### Test plan #### Successful Creation of an encrypted volume - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create the secret for that volume in the configured namespace - create a pvc that references the created storage class - create a pod that uses that pvc for a volume mount - wait for pod up and healthy #### Successful Creation of an encrypted volume with customization of the cipher - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create the secret with customized options of the cipher for that volume in the configured namespace - create a pvc that references the created storage class - create a pod that uses that pvc for a volume mount - wait for pod up and healthy - check if the customized options of the cipher are correct #### Missing Secret for encrypted volume creation - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create a pvc that references the created storage class - create a pod that uses that pvc for a volume mount - verify pvc remains in pending state - verify pod remains in creation state #### Verify encryption of volume - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create the secret for that volume in the configured namespace - create a pvc that references the created storage class - create a pod that uses that pvc for a volume mount - wait for pod up and healthy - write known test pattern into fs - verify absence (grep) of known test pattern after reading block device content `/dev/longhorn/` #### Verify wrong key failure - create a storage class with (encrypted=true) and either a global secret or a per volume secret - create the secret for that volume in the configured namespace - create a pvc that references the created storage class - create a pod that uses that pvc for a volume mount - wait for pod up and healthy - scale down pod - change `CRYPTO_KEY_VALUE` of secret - scale up pod - verify pod remains in pending state (failure to mount volume) ### Upgrade strategy - requires new pvc's since encryption would overwrite the previously created filesystem. (csi driver prevents this) ## Note - Host requires `dm_crypt` kernel module as well as `cryptsetup` installed. - [csi sidecars and secrets](https://kubernetes-csi.github.io/docs/secrets-and-credentials.html) - supporting external key vaults is possible in the future with some additional implementation - support rotating keys is possible in the future with some additional implementation ================================================ FILE: enhancements/20221103-filesystem-trim.md ================================================ # Filesystem Trim ## Summary Longhorn can reclaim disk space by allowing the filesystem to trim/unmap the unused blocks occupied by removed files. ### Related Issues https://github.com/longhorn/longhorn/issues/836 ## Motivation ### Goals 1. Longhorn volumes support the operation `unmap`, which is actually the filesystem trim. 2. Since some unused blocks are in the snapshots, these blocks can be freed if the snapshots is no longer required. ## Proposal 1. Longhorn tgt should support module `UNMAP`. When the filesystem of a Longhorn volume receives cmd `fstrim`, the iSCSI initiator actually sends this `UNMAP` requests to the target. To understand the iscsi protocol message of `UNMAP` then start the implementation, we can refer to Section "3.54 UNMAP command" of [the doc](https://www.seagate.com/files/staticfiles/support/docs/manual/Interface%20manuals/100293068j.pdf). 2. By design, snapshots of a Longhorn volume are immutable and lots of the blocks of removed files may be in the snapshots. This implicitly means we have to skip these blocks and free blocks in the current volume head only if we do nothing else. It will greatly degrade the effectiveness of this feature. To release as much space as possible, we can do unmap for all continuous unavailable (removed or system) snapshots behinds the volume head, which is similar to the snapshot prune operation. 3. Longhorn volumes won't mark snapshots as removed hence most of the time there is no continuous unavailable snapshots during the trim. To make it more practicable, we introduce a new global setting for all volumes. It automatically marks the latest snapshot and its ancestors as removed and stops at the snapshot containing multiple children. Besides, there is a per-volume option that can overwrite the global setting and directly indicate if this automatic removal is enabled. By default, it will be ignored and the volumes follow the global setting. ### User Stories #### Reclaim the space wasted by the removed files in a filesystem Before the enhancement, there is no way to reclaim the space. To shrink the volume, users have to launch a new volume with a new filesystem, and copy the existing files from the old volume filesystem to the new one, then switch to use the new volume. After the enhancement, users can directly reclaim the space by trimming the filesystem via cmd `fstrim` or Longhorn UI. Besides, users can enable the new option so that Longhorn can automatically mark the snapshot chain as removed then trim the blocks recorded in the snapshots. ### User Experience In Detail 1. Users can enable the option for a specific volume by modifying the volume option `volume.Spec.UnmapMarkSnapChainRevmoed`, or directly set the global setting `Remove Snapshots During Filesystem Trim`. 2. For an existing Longhorn volume that contains a filesystem and there are files removed from the filesystem, users can directly run cmd `fstrim ` or Click Longhorn UI button `Trim Filesystem`. 3. Users will observe that the snapshot chain are marked as removed. And both these snapshots and the volume head will be shrunk. ### API Changes - Volume APIs: - Add `updateUnmapMarkSnapChainRemoved`: Control if Longhorn will remove snapshots during the filesystem trim, or just follows the global setting. - Add `trimFilesystem`: Trim the filesystem of the volume. Best Effort. - Engine APIs: - Add `unmap-mark-snap-chain-removed`: `--enable` or `disable`. Control if the engine and all its replicas will mark the snapshot chain as removed once receiving a `UNMAP` request. ## Design ### Implementation Overview #### longhorn-manager: 1. Add a setting `Remove Snapshots During Filesystem Trim`. 2. Add fields for CRDs: `volume.Spec.UnmapMarkSnapChainRemoved`, `engine.Spec.UnmapMarkSnapChainRemoved`, `replica.Spec.UnmapMarkDiskChainRemoved`. 3. Add 2 HTTP APIs mentioned above: `updateUnmapMarkSnapChainRemoved` and `trimFilesystem`. 4. Update controllers . 1. `Volume Controller`: 1. Update the engine and replica field based on `volume.Spec.UnmapMarkSnapChainRemoved` and the global setting. 2. Enqueue the change for the field and the global setting. 2. `Engine Controller`: 1. The monitor thread should compare the CR field `engine.Spec.UnmapMarkSnapChainRemoved` with the current option value inside the engine process, then call the engine API `unmap-mark-snap-chain-removed` if there is a mismatching. 2. The process creation function should specify the option `unmap-mark-snap-chain-removed`. 3. `Replica Controller`: 1. The process creation function should specify the option `unmap-mark-disk-chain-removed`. #### longhorn-engine: 1. Update dependency `rancher/tgt`, `longhorn/longhornlib`, and `longhorn/sparse-tools` for the operation `UNMAP` support. 2. Add new option `unmap-mark-snap-chain-removed` for the engine process creation call. Add new option `unmap-mark-disk-chain-removed` for the replica process creation call. 3. Add a new API `unmap-mark-snap-chain-removed` to update the field for the engine and all its replicas. 4. The engine process should be able to recognize the request of `UNMAP` from the tgt, then forwards the requests to all its replicas via the dataconn service. This is similar to data R/W. 5. When each replica receive a trim/unmap request, it should decide if the snapshot chain can be marked as removed, then collect all trimmable snapshots, punch holes to these snapshots and the volume head, then calculate the trimmed space. #### instance-manager: - Update the dependencies. - Add the corresponding proxy API for the new engine API. #### longhorn-ui: 1. Add 2 new operations for Longhorn volume. - API `updateUnmapMarkSnapChainRemoved`: - The backend accepts 3 values of the input `UnmapMarkSnapChainRemoved`: `"enabled"`, `"disabled"`, `"ignored"`. - The UI can rename this option to `Remove Current Snapshot Chain during Filesystem Trim`, and value `"ignored"` to `follow the global setting`. - API `trimFilesystem`: No input is required. 2. The volume creation call accepts a new option `UnmapMarkSnapChainRemoved`. This is almost the same as the above update API. ### Test Plan #### Integration tests Test if the unused blocks in the volume head and the snapshots can be trimmed correctly without corrupting other files, and if the snapshot removal mechanism works when the option is enabled or disabled. #### Manual tests Test if the filesystem trim works correctly when there are continuous writes into the volume. ### Upgrade strategy N/A ================================================ FILE: enhancements/20221109-support-bundle-enhancement.md ================================================ # Support Bundle Enhancement ## Summary This feature replaces the support bundle mechanism with the general purpose [support bundle kit](https://github.com/rancher/support-bundle-kit). Currently, the Longhorn support bundle file is hard to understand, and analyzing it is difficult. With the new support bundle, the user can simulate a mocked Kubernetes cluster that is interactable with `kubectl`. Hence makes the analyzing process more intuitive. ### Related Issues https://github.com/longhorn/longhorn/issues/2759 ## Motivation ### Goals - Replace the Longhorn support-bundle generation mechanism with the support bundle manager. - Keep the same support bundle HTTP API endpoints. - Executing the` support-bundle-kit simulator` on the support bundle can start a mocked Kubernetes API server that is interactable using `kubectl`. - Introduce a new `support-bundle-manager-image` setting for easy support-bundle manager image replacement. - Introduce a new `support-bundle-failed-history-limit` setting to avoid unexpected increase of failed support bundles. ### Non-goals [optional] `None` ## Proposal - Introduce the new `SupportBundle` custom resource definition. - Creating a new custom resource triggers the creation of a support bundle manager deployment. The support-bundle manager is responsible for support bundle collection and exposes it to `https://:8080/bundle`. - Deleting the SupportBundle custom resource deletes its owning support bundle manager deployment. - Introduce a new `longhorn-support-bundle` controller. - Responsible for SupportBundle custom resource status updates and event recordings. - [The controller reacts in phases based on the SupportBundle state.](#manager-supportbundle-creation-handling-in-longhorn-support-bundle-controller) - Responsible for cleaning up the support bundle manager deployment when the owner `SupportBundle` custom resource is tagged for deletion. - There is no change to the HTTP API endpoints. This feature replaces the handler function logic. - Introduce a new `longhorn-support-bundle` service account with `cluster-admin` access. The current `longhorn-service-account` service account cannot generate the following resources. ``` Failed to get /api/v1/componentstatuses Failed to get /apis/authentication.k8s.io/v1/tokenreviews Failed to get /apis/authorization.k8s.io/v1/selfsubjectrulesreviews Failed to get /apis/authorization.k8s.io/v1/subjectaccessreviews Failed to get /apis/authorization.k8s.io/v1/selfsubjectaccessreviews Failed to get /apis/certificates.k8s.io/v1/certificatesigningrequests Failed to get /apis/networking.k8s.io/v1/ingressclasses Failed to get /apis/policy/v1beta1/podsecuritypolicies Failed to get /apis/rbac.authorization.k8s.io/v1/clusterroles Failed to get /apis/rbac.authorization.k8s.io/v1/clusterrolebindings Failed to get /apis/node.k8s.io/v1/runtimeclasses Failed to get /apis/flowcontrol.apiserver.k8s.io/v1beta1/prioritylevelconfigurations Failed to get /apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas Failed to get /api/v1/namespaces/default/replicationcontrollers Failed to get /api/v1/namespaces/default/bindings Failed to get /api/v1/namespaces/default/serviceaccounts Failed to get /api/v1/namespaces/default/resourcequotas Failed to get /api/v1/namespaces/default/limitranges Failed to get /api/v1/namespaces/default/podtemplates Failed to get /apis/apps/v1/namespaces/default/replicasets Failed to get /apis/apps/v1/namespaces/default/controllerrevisions Failed to get /apis/events.k8s.io/v1/namespaces/default/events Failed to get /apis/authorization.k8s.io/v1/namespaces/default/localsubjectaccessreviews Failed to get /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers Failed to get /apis/networking.k8s.io/v1/namespaces/default/ingresses Failed to get /apis/networking.k8s.io/v1/namespaces/default/networkpolicies Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/default/rolebindings Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/default/roles Failed to get /apis/storage.k8s.io/v1beta1/namespaces/default/csistoragecapacities Failed to get /apis/discovery.k8s.io/v1/namespaces/default/endpointslices Failed to get /apis/helm.cattle.io/v1/namespaces/default/helmcharts Failed to get /apis/helm.cattle.io/v1/namespaces/default/helmchartconfigs Failed to get /apis/k3s.cattle.io/v1/namespaces/default/addons Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/ingressroutetcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/ingressroutes Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/serverstransports Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/traefikservices Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/middlewaretcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/middlewares Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/tlsstores Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/tlsoptions Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/default/ingressrouteudps Failed to get /api/v1/namespaces/kube-system/bindings Failed to get /api/v1/namespaces/kube-system/resourcequotas Failed to get /api/v1/namespaces/kube-system/serviceaccounts Failed to get /api/v1/namespaces/kube-system/podtemplates Failed to get /api/v1/namespaces/kube-system/limitranges Failed to get /api/v1/namespaces/kube-system/replicationcontrollers Failed to get /apis/apps/v1/namespaces/kube-system/controllerrevisions Failed to get /apis/apps/v1/namespaces/kube-system/replicasets Failed to get /apis/events.k8s.io/v1/namespaces/kube-system/events Failed to get /apis/authorization.k8s.io/v1/namespaces/kube-system/localsubjectaccessreviews Failed to get /apis/autoscaling/v1/namespaces/kube-system/horizontalpodautoscalers Failed to get /apis/networking.k8s.io/v1/namespaces/kube-system/networkpolicies Failed to get /apis/networking.k8s.io/v1/namespaces/kube-system/ingresses Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/kube-system/rolebindings Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/kube-system/roles Failed to get /apis/storage.k8s.io/v1beta1/namespaces/kube-system/csistoragecapacities Failed to get /apis/discovery.k8s.io/v1/namespaces/kube-system/endpointslices Failed to get /apis/helm.cattle.io/v1/namespaces/kube-system/helmchartconfigs Failed to get /apis/helm.cattle.io/v1/namespaces/kube-system/helmcharts Failed to get /apis/k3s.cattle.io/v1/namespaces/kube-system/addons Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/serverstransports Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/middlewaretcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/middlewares Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/tlsstores Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/ingressrouteudps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/ingressroutes Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/ingressroutetcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/traefikservices Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/kube-system/tlsoptions Failed to get /api/v1/namespaces/cattle-system/limitranges Failed to get /api/v1/namespaces/cattle-system/podtemplates Failed to get /api/v1/namespaces/cattle-system/resourcequotas Failed to get /api/v1/namespaces/cattle-system/serviceaccounts Failed to get /api/v1/namespaces/cattle-system/replicationcontrollers Failed to get /api/v1/namespaces/cattle-system/bindings Failed to get /apis/apps/v1/namespaces/cattle-system/replicasets Failed to get /apis/apps/v1/namespaces/cattle-system/controllerrevisions Failed to get /apis/events.k8s.io/v1/namespaces/cattle-system/events Failed to get /apis/authorization.k8s.io/v1/namespaces/cattle-system/localsubjectaccessreviews Failed to get /apis/autoscaling/v1/namespaces/cattle-system/horizontalpodautoscalers Failed to get /apis/networking.k8s.io/v1/namespaces/cattle-system/networkpolicies Failed to get /apis/networking.k8s.io/v1/namespaces/cattle-system/ingresses Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/cattle-system/roles Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/cattle-system/rolebindings Failed to get /apis/storage.k8s.io/v1beta1/namespaces/cattle-system/csistoragecapacities Failed to get /apis/discovery.k8s.io/v1/namespaces/cattle-system/endpointslices Failed to get /apis/helm.cattle.io/v1/namespaces/cattle-system/helmchartconfigs Failed to get /apis/helm.cattle.io/v1/namespaces/cattle-system/helmcharts Failed to get /apis/k3s.cattle.io/v1/namespaces/cattle-system/addons Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/tlsoptions Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/traefikservices Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/middlewares Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/ingressroutetcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/serverstransports Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/ingressroutes Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/middlewaretcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/tlsstores Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/cattle-system/ingressrouteudps Failed to get /api/v1/namespaces/longhorn-system/limitranges Failed to get /api/v1/namespaces/longhorn-system/podtemplates Failed to get /api/v1/namespaces/longhorn-system/resourcequotas Failed to get /api/v1/namespaces/longhorn-system/replicationcontrollers Failed to get /api/v1/namespaces/longhorn-system/serviceaccounts Failed to get /api/v1/namespaces/longhorn-system/bindings Failed to get /apis/apps/v1/namespaces/longhorn-system/replicasets Failed to get /apis/apps/v1/namespaces/longhorn-system/controllerrevisions Failed to get /apis/events.k8s.io/v1/namespaces/longhorn-system/events Failed to get /apis/authorization.k8s.io/v1/namespaces/longhorn-system/localsubjectaccessreviews Failed to get /apis/autoscaling/v1/namespaces/longhorn-system/horizontalpodautoscalers Failed to get /apis/networking.k8s.io/v1/namespaces/longhorn-system/ingresses Failed to get /apis/networking.k8s.io/v1/namespaces/longhorn-system/networkpolicies Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/longhorn-system/rolebindings Failed to get /apis/rbac.authorization.k8s.io/v1/namespaces/longhorn-system/roles Failed to get /apis/storage.k8s.io/v1beta1/namespaces/longhorn-system/csistoragecapacities Failed to get /apis/discovery.k8s.io/v1/namespaces/longhorn-system/endpointslices Failed to get /apis/helm.cattle.io/v1/namespaces/longhorn-system/helmchartconfigs Failed to get /apis/helm.cattle.io/v1/namespaces/longhorn-system/helmcharts Failed to get /apis/k3s.cattle.io/v1/namespaces/longhorn-system/addons Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/serverstransports Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/ingressroutes Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/tlsstores Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/traefikservices Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/tlsoptions Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/middlewares Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/ingressroutetcps Failed to get /apis/traefik.containo.us/v1alpha1/namespaces/longhorn-system/middlewaretcps ``` ### User Stories #### Support Bundle Generation This feature does not alter how the user generates the support bundle on UI. #### Mocking Support Bundle Cluster The user can simulate a mocked cluster with the support bundle and interact using `kubectl`. ### User Experience In Detail #### Support Bundle Generation 1. User clicks `Generate Support BundleFile` in Longhorn UI. 1. Longhorn creates a `SupportBundle` custom resource. 1. Longhorn creates a support bundle manager deployment. 1. User downloads the support bundle as same as before. 1. Longhorn deletes the `SupportBundle` custom resource. 1. Longhorn deletes the support bundle manager deployment. #### Support Bundle Generation Failed 1. User clicks `Generate Support BundleFile` in Longhorn UI. 1. Longhorn creates a SupportBundle custom resource. 1. Longhorn creates a support bundle manager deployment. 1. The SupportBundle goes into an error state. 1. User sees an error on UI. 1. Longhorn retains the failed SupportBundle and its support-bundle manager deployment. 1. User analyzes the failed SupportBundle on the cluster. Or generate a new support bundle so the failed SupportBundle can be analyzed off-site. 1. User deletes the failed SupportBundle when done with the analysis. Or have Longhorn automatically purge all failed SupportBundles by setting [support bundle failed history limit](#manager-support-bundle-failed-history-limit-setting) to 0. 1. Longhorn deletes the SupportBundle custom resource. 1. Longhorn deletes the support bundle manager deployment. ### API changes #### Longhorn manager HTTP API There will be no change to the HTTP API endpoints. This feature replaces the handler function logic. | Method | Path | Description | | -------- | ------------------------------------------------- | ----------------------------------------------------- | | **POST** | `/v1/supportbundles` | Creates SupportBundle custom resource | | **GET** | `/v1/supportbundles/{name}/{bundleName}` | Get the support bundle details from the SupportBundle custom resource | | **GET** | `/v1/supportbundles/{name}/{bundleName}/download` | Get the support bundle file from `https://:8080/bundle` | ## Design ### Implementation Overview #### Deployment: longhorn-support-bundle service account Collecting the support bundle requires complete cluster access. Hence Longhorn will have a service account dedicated at deployment. ```yaml --- apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-support-bundle namespace: longhorn-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-support-bundle roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: longhorn-support-bundle namespace: longhorn-system --- ``` #### Manager: HTTP API SupportBundle resource ```go type SupportBundle struct { client.Resource NodeID string `json:"nodeID"` State longhorn.SupportBundleState `json:"state"` Name string `json:"name"` ErrorMessage string `json:"errorMessage"` ProgressPercentage int `json:"progressPercentage"` } ``` #### Manager: POST `/v1/supportbundles` - Creates a new `SupportBundle` custom resource. #### Manager: GET `/v1/supportbundles/{name}/{bundleName}` - Gets the `SupportBundle` custom resource and returns [SupportBundle resource](#manager-http-api-supportbundle-resource). #### Manager: GET `/v1/supportbundles/{name}/{bundleName}/download` 1. Get the support bundle from [https://\:8080/bundle](https://github.com/rancher/support-bundle-kit/blob/master/pkg/manager/httpserver.go#L104). 1. Copy the support bundle to the response writer. 1. Delete the `SupportBundle` custom resource. #### Manager: SupportBundle custom resource ```yaml apiVersion: v1 items: - apiVersion: longhorn.io/v1beta2 kind: SupportBundle metadata: creationTimestamp: "2022-11-10T02:35:45Z" generation: 1 name: support-bundle-2022-11-10t02-35-45z namespace: longhorn-system resourceVersion: "97016" uid: a5169448-a6e5-4637-b99a-63b9a9ea0b7f spec: description: "123" issueURL: "" nodeID: "" status: conditions: - lastProbeTime: "" lastTransitionTime: "2022-11-10T03:35:29Z" message: done reason: Create status: "True" type: Manager filename: supportbundle_08ccc085-641c-4592-bb57-e05456241204_2022-11-10T02-36-13Z.zip filesize: 502608 image: rancher/support-bundle-kit:master-head managerIP: 10.42.2.54 ownerID: ip-10-0-1-113 progress: 100 state: ReadyForDownload kind: List metadata:master-head resourceVersion: "" ``` #### Manager: `support-bundle-manager-image` setting The support bundle manager image for the support bundle generation. ``` Category = general Type = string Default = rancher/support-bundle-kit:master-head ``` #### Manager `support-bundle-failed-history-limit` setting This setting specifies how many failed support bundles can exist in the cluster. The retained failed support bundle is for analysis purposes and needs to clean up manually. Set this value to 0 to have Longhorn automatically purge all failed support bundles. ``` Category = general Type = integer Default = 1 ``` #### Manager: validate at SupportBundle creation 1. Block creation if the number of failed SupportBundle exceeds the [support bundle failed history limit](#manager-support-bundle-failed-history-limit-setting). 1. Block creation if there is another SupportBundle is in progress. However, skip checking the SupportBundle that is in an error state. We will leave the user to decide what to do with the failed SupportBundles. #### Manager: mutate at SupportBundle creation 1. Add finalizer. #### Manager: SupportBundle creation handling in longhorn-support-bundle controller This controller handles the support bundle in phases depending on its custom resource state. At the end of each phase will update the SupportBundle custom resource state and then returns the queue. The controller picks up the update and enqueues again for the next phase. When there is no state update, the controller automatically queues the handling custom resource until the state reaches `ReadyForDownload` or `Error`. **State: None("")** - Update the custom resource image with the setting value. - Update the custom resource state to `Started`. **State: Started** - Update the state to `Generating` when the support bundle manager deployment exists. - Create support bundle manager deployment and requeue this phase to check support bundle manager deployment. **State: Generating** - Update the [SupportBundle](#manager-supportbundle-custom-resource) status base on the support manager [https://\:8080/status](https://github.com/rancher/support-bundle-kit/blob/master/pkg/manager/httpserver.go#L103): - IP - file name - progress - filesize - Update the custom resource state to `ReadyForDownload` when progress reached 100. #### Manager: SupportBundle error handling in longhorn-support-bundle controller - Update the state to `Error` and record the error type condition when the phase encounters unexpected failure. - When the [support bundle failed history limit](#manager-support-bundle-failed-history-limit-setting) is 0, update the state to `Purging`. **Purging** - Delete all failed SupportBundles in the state `Error`. #### Manager: SupportBundle deletion handling in longhorn-support-bundle controller When the SupportBundle gets marked with `DeletionTimestamp`, the controller updated its state to `Deleting`. **Deleting** - Delete its support bundle manager deployment. - Remove the SupportBundle finalizer. #### Manager: SupportBundle purge handling in longhorn-setting controller - If the [support bundle failed history limit](#manager-support-bundle-failed-history-limit-setting) is 0, update all failed SupportBundle state to `Purging`. ### Test plan - Test support bundle generation should be successful. - Test support bundle should be cleaned up after download. - Test support bundle should retain when generation failed. - Test support bundle should generate when the cluster has an existing `SupportBundle` in an error state. - Test support bundle should purge when `support bundle failed history limit` is set to 0. - Test support bundle cluster simulation. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20221123-local-volume.md ================================================ # Local Volume ## Summary Longhorn can support local volume to provide better IO latencies and IOPS. ### Related Issues https://github.com/longhorn/longhorn/issues/3957 ## Motivation ### Goals - Longhorn can support local volume (data locality=strict-local) for providing better IO latencies and IOPS. - A local volume can only have one replica. - A local volume supports the operations such as snapshot, backup and etc. ### Non-goals - A local volume's data locality cannot be converted to other modes when volume is not detached. - A local volume does not support multiple replicas in the first version. The local replication could be an improvement in the future. ## Proposal 1. Introduce a new type of volume type, a local volume with `strict-local` data locality. - Different than a volume with `best-effort` data locality, the engine and replica of a local volume have to be located on the same node. 2. Unix-domain socket are used instead of TCP between the replica process' data server and the engine. 3. A local volume supports the existing functionalities such as snapshotting, backup, restore, etc. ### User Stories Longhorn is a highly available replica-based storage system. As the data path is designed for the replication, a volume with a single replica still suffers from high IO latency. In some cases, the distributed data workloads such as databases already have their own data replication, sharding, etc, so we should provide a volume type for these use cases while supporting existing volume functionalities like snapshotting, backup/restore, etc. ### User Experience In Detail - The functionalities and behaviors of the volumes with `disabled` and `best-effort` data localities will not be changed. - A volume with `strict-local` data locality - Only has one replica - The engine and replica have to be located on the same node - Cannot convert to `disabled` or `best-effort` data locality when the volume is not detached - Can convert to `disabled` or `best-effort` data locality when the volume is detached - Existing functionalities such as snapshotting, backup, restore, etc. are supported ### CLI Changes - Add `--volume-name` in engine-binary `replica` command - The unix-domain-socket file will be `/var/lib/longhorn/unix-domain-socket/${volume name}.sock` - Add `--data-server-protocol` in engine-binary `replica` command - Available options are `tcp` (default) and `unix` - Add `--data-server-protocol` in engine-binary `controller` command - Available options are `tcp` (default) and `unix` ## Design ### Implementation Overview #### CRDs 1. Add a new data locality `strict-local` in `volume.Spec.DataLocality` #### Volume Creation and Attachment - When creating and attaching a volume with `strict-local` data locality, the replica is scheduled on the node where the engine is located. - Afterward, the replica process is created with the options `--volume-name ${volume name}` and `--data-server-protocol unix`. - The data server in the replica process is created and listens on a unix-domain-socket file (`/var/lib/longhorn/unix-domain-socket/${volume name}.sock`). - Then, the engine process of the volume is created with the option `--data-server-protocol unix`. - The client in the engine process connects to the data server in the replica process via the unix-domain-socket file. ### Validating Webhook - If a volume with `strict-local` data locality, the `numberOfReplicas` should be 1. - If a local volume is attached, the conversion between `strict-local` and other data localities is not allowable. - If a local volume is attached, the update of the replica count is not allowable. ### Test Plan #### Integration tests 1. Successfully create a local volume with `numberOfReplicas=1` and `dataLocality=strict-local`. 2. Check the validating webhook can reject the following cases when the volume is created or attached - Create a local volume with `dataLocality=strict-local` but `numberOfReplicas>1` - Update a attached local volume's `numberOfReplicas` to a value greater than one - Update a attached local volume's `dataLocality` to `disabled` or `best-effort` ================================================ FILE: enhancements/20221205-concurrent-backup-restore-limit.md ================================================ # Concurrent Backup Restore Per Node Limit ## Summary Longhorn has no boundary on the number of concurrent volume backup restoring. Having a new `concurrent-backup-restore-per-node-limit` setting allows the user to limit the concurring backup restoring. Setting this restriction lowers the potential risk of overloading the cluster when volumes restoring from backup concurrently. For ex: during the Longhorn system restore. ### Related Issues https://github.com/longhorn/longhorn/issues/4558 ## Motivation ### Goals Introduce a new `concurrent-backup-restore-per-node-limit` setting to define the boundary of the concurrent volume backup restoring. ### Non-goals `None` ## Proposal 1. Introduce a new `concurrent-backup-restore-per-node-limit` setting. 1. Track the number of per-node volumes restoring from backup with atomic count (thread-safe) in the engine monitor. ### User Stories Allow the user to set the concurrent backup restore per node limit to control the risk of cluster overload when Longhorn volume is restoring from backup concurrently. ### User Experience In Detail 1. Longhorn holds the engine backup restore when the number of volume backups restoring on a node reaches the `concurrent-backup-restore-per-node-limit`. 1. The volume backup restore continues when the number of volume backups restoring on a node is below the limit. ## Design ### Implementation Overview #### The `concurrent-backup-restore-per-node-limit` Setting This setting controls how many engines on a node can restore the backup concurrently. Longhorn engine monitor backs off when the volume [backup restoring count](#track-the-volume-backup-restoring-per-node) reaches the setting limit. Set the value to **0** to disable backup restore. ``` Category = SettingCategoryGeneral, Type = integer Default = 5 # same as the default replica rebuilding number ``` #### Track the volume backup restoring per node 1. Create a new atomic counter in the engine controller. ``` type EngineController struct { restoringCounter util.Counter } ``` 1. Pass the restoring counter to each of its engine monitors. ``` type EngineMonitor struct { restoringCounter util.Counter } ``` 1. Increase the restoring counter before backup restore. > Ignore DR volumes (volume.Status.IsStandby). 1. Decrease the restoring counter when the backup restore caller method ends ### Test plan - Test the setting should block backup restore when creating multiple volumes from the backup at the same time. - Test the setting should be per-node limited. - Test the setting should not have effect on DR volumes. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20221213-reimplement-longhorn-engine-with-SPDK.md ================================================ # Reimplement Longhorn Engine with SPDK ## Summary The Storage Performance Development Kit [SPDK](https://spdk.io) provides a set of tools and C libraries for writing high performance, scalable, user-mode storage applications. It achieves high performance through the use of a number of key techniques: * Moving all of the necessary drivers into userspace, which avoids syscalls and enables zero-copy access from the application. * Polling hardware for completions instead of relying on interrupts, which lowers both total latency and latency variance. * Avoiding all locks in the I/O path, instead relying on message passing. SPDK has several features that allow it to perform tasks similar to what the `longhorn-engine` currently needs: * [Block Device](https://spdk.io/doc/bdev.html) layer, often simply called bdev, intends to be equivalent to the operating system block storage layer that often sits immediately above the device drivers in a traditional kernel storage stack. SPDK provides also virtual bdev modules which creates block devices on existing bdev, for example Logical Volumes or RAID1. * [Logical volumes](https://spdk.io/doc/logical_volumes.html) library is a flexible storage space management system. It allows creating and managing virtual block devices with variable size on top of other bdevs. The SPDK Logical Volume library is built on top [Blobstore](https://spdk.io/doc/blob.html) which is a persistent, power-fail safe block allocator designed to be used as the local storage system backing a higher level storage service, typically in lieu of a traditional filesystem. Logical volumes have a couple of features like Thinly Provisioning and Snapshots similar to what actual Longhorn-Engine provides. * [NVMe over Fabrics](https://spdk.io/doc/nvmf.html) is a feature to presents block devices over a fabrics such as Ethernet, supporting RDMA and TCP transports. The standard Linux kernel initiators for NVMe-oF interoperate with these SPDK NVMe-oF targets, so with this feature we can serve bdev over the network or to other processes ## Motivation These are the reasons that have driven us: * Use SPDK to improve performance of Longhorn * Use SPDK functionality to improve reliability and robustness * Use SPDK to take advantage of the new features that are continuously added to the framework ### Goals * Implement all actual `longhorn-engine` functionalities * Continue to support multiple `longhorn-engine` versions concurrently * Maintain as much as possible the same user experience between Longhorn with and without SPDK * Lay the groundwork for extending Longhorn to sharding and aggegration of storage devices ## Proposal SPDK implements a JSON-RPC 2.0 server to allow external management tools to dynamically configure SPDK components ([documentation](https://spdk.io/doc/jsonrpc.html)). What we aim is to create an external orchestrator that, with JSON-RPC calls towards multiple instances of `spdk_tgt` app running in different machines, could manage the durability and reliability of data. Actually, not all needed functionalities to do that are already available in SPDK, so some new JSON-RPC commands will be developed over SPDK. This orchestrator is implemented in longhorn manager pods and will use a new process, called `longhorn-spdk-engine` in continuity with actual `longhorn-engine`, to talk with `spdk_tgt`. * The main purpose of `longhorn-spdk-engine` is to create and export via NVMe-oF logical volumes from multiple replica nodes (one of them likely local), attach to these volumes on a controller node, use resulting bdevs to create a RAID1 bdev and exporting it via NVMe-oF locally. At this point NVMe Kernel module can be used to connect to this NVMe-oF subsystem and so to create a block device `/dev/nvmeXnY` to be used by the Longhorn CSI driver. In this way we will have multiple replica of the same data written on this block device. * Below a diagram that shows the control plane of the proposal ![SPDK New Architecture](./image/spdk-control-plane.png) * In release 23.01, support for ublk will be added in SPDK: with this functionality we can directly create a block device without using the NVMe layer on Linux kernel versions >6.0. This will be a quite big enhancement over using NVMe-oF locally. The `longhorn-spdk-engine` will be responsible to make all others control operations, like for example creating snapshots over all replicas of the same volume. Other functionalities orchestrated through the engine will be the remote rebuild, a complete rebuild of the entire snapshot stack of a volume needed to add or repair a replica, the backup and restore, export/import of a SPDK logical volumes to/from sparse files stored on an external storage system via S3. The `longhorn-spdk-engine` will be developed in Go so maybe we can reuse some code from `longhorn-engine`, for example gRPC handling to receive control commands and error handling during snapshot/backup/restore operations. What about the data plane, below a comparison between actual architecture and new design: * longhorn-engine ![](./image/engine-data-plane.png) * spdk_tgt ![](./image/spdk-data-plane.png) ## Design ### Implementation Overview Actually there is a `longhorn-engine` controller and some `longhorn-engine` replica for every volume to manage. All these instances are started and controlled by the `instance-manager`, so on every node belonging to the cluster we have one instance of `instance-manager` and multiple instances of `longhorn-engine`. Every volume is stored in a sequence of sparse files representing the live data and the snapshots. With SPDK we have a different situation, because `spdk_tgt` can take the control of an entire disk, so in every node we will have a single instance of SPDK that will handle all the volumes created by Longhorn. To orchestrate SPDK instances running on different nodes in a way to make up a set of replicas, we will introduce, as discussed before, the `longhorn-spdk-engine`; to make the volume management lighter we will have an instance of the engine per volume. `longhorn-spdk-engine` will implement actual gRPC interface used by `longhorn-engine` to talk with `instance-manager`, so this last one will became the portal to communicate with `longhorn-manager` by different data plane. `spdk_tgt` by default starts with a single thread, but it can be configured to use multiple threads: we can have a thread per core available on the CPU. This will increase the performance but comes with the cost of an high CPU utilization. Working in polling mode instead than in interrupt mode, CPU core utilization by a single thread is always rising 100% even with no workload to handle. This could be a problem, so we can configure `spdk_tgt` with dynamic scheduler: in this way, if no workload is present, only one core will be used and only one thread will continue polling. Other thread will be put in a idle state and will become active again only when needed. Moreover, dynamic scheduler has a way to reduce the CPU frequency. (See future work section.) ### Snapshots When `longhorn-spdk-engine` receive a snapshot request from `instance-manager`, before to proceed all I/O operations over volume's block device `/dev/nvmeXnY` must be stopped to ensure that snapshots over all the replicas contains the same data. Actually there is no way to suspend the I/O operations over a block device, so we will have to implement this feature into SPDK. But in RAID bdev there are already some private functions to suspend I/O (they will be used for example in base bdev removing), maybe we can use and improve them. These functions actually enqueue all the I/O operations received during the suspend time. Once received a snapshot request, `longhorn-spdk-engine` will call the JSON-RPC to make a snapshot over the local replica of the volume involved. The snapshot RPC command will ensure to freeze all I/O over the logical volume to be snapshotted, so all pending I/O will be executed before the snapshot. SPDK logical volume have a couple of features that we will use: * clone, used to create new logical volume based on a snapshot. It can be used to revert a volume to a snapshot too, cloning a new volume, deleting the old one and then renaming the new one as the old one * decouple, feature that can be used to delete a snapshot, first decoupling the child volume from this snapshot and then deleting the snapshot. ### Replica rebuild RAID replica rebuild is actually under development, so we don'know exactly hot it will be implemented, but we can suppose that we will not use it because presumably it will work only at bdev layer. When a new replica has to be added or a replica has to be rebuilt, we have to recreate the entire snapshot stack of each volume that are hosted on that node. Actually SPDK doesn't have nothing to do that, but after discussing with core maintainers we arranged a procedure. Let's make an example. Supposing we have to rebuild a volume with two layer of snapshots, snapshotA is the oldest and snapshotB the younger, basically we have to (in _italic_ what we miss): * create a new volume on the node to be rebuilt * export this volume via NVMe-oF * attach to this volume in the node where we have the source data * _copy snapshotA over the attached volume_ * perform a snapshot over the exported volume * repeat the copy and snapshot operations for snapshotB What we have to implement is a JSON-RPC to copy a logical volume over an arbitrary bdev (that in our case will represent a remote volume exported via NVMe-oF and locally attached) _while the top layer is also being modified_ (see next section). So, in this way we can rebuild the snapshot stack of a volume. But what about the live data? Actual `longhorn-engine` make the replica rebuild in an "hot" way, i.e., during the rebuilding phase it is writing over the live data of the new volume. So, how can we reproduce this with SPDK? First of all we have to wait the conclusion of RAID1 bdev's review process to see what kind of replica rebuild will be implemented. But, supposing that the rebuild feature will not be useful for us, we will need to create a couple of additional JSON-RPC over SPDK to implement the following procedure (in _italic_ what we miss): * create a new volume over the node to be rebuilt * export this volume via NVMe-oF * attach to this volume in the node where we have the RAID1 * _add the bdev of the attached volume to the RAID1 bdev excluded from the read balancing_ * wait for the snapshot stack rebuilding to finish * _change the upper volume of the snapshot stack from the current to this one with the live data_ * _enable the bdev of the attached volume for RAID1 read balancing_ What we have at the end of the rebuilding's phase is a snapshot stack with an empty volume at the top, while in the RAID1 we have a volume with the live data but without any snapshot. So we have to couple these 2 stacks exchanging the upper volume and to do that we need a new JSON-RPC. We will need to implement the JSON-RPC to enable/disable a bdev from the RAID1 read balancing too. ### Backup and Restore Backup will be implemented exporting a volume to a sparse file and then save this file over an external storage via S3. SPDK already has a `spdk_dd` application that can copy a bdev to a file and this app has an option to preserve bdev sparseness. But using spdk_dd has some problems: actually the sparse option works only with bdev that represent a local logical volume, not an exported one via NVMe-oF. So to backup a volume we cannot work on a remote node where to export this volume, we need to work on the node where we have the data source. But in this way, to perform a backup, we would need to stop the `spdk_tgt` app, run the `spdk_dd` and then restart the `spdk_tgt`. This operation is needed because it could not be safe to run multiple spdk applications over the same disk (even if spdk_dd would read from a read only volume) and moreover `spdk_dd` could not see the volume to export if this has been created after the last restart of `spdk_tgt` app. This because blobstore metadata, and so newly created logical volume, are saved on disk only on application exit. Stopping `spdk_tgt` is not acceptable because it would suspend operation over all other volumes hosted in this node so, to solve these problems, we have 2 possible solutions: * create a JSON-RPC command to export logical volume to a sparse file, so that we can make the operation directly over the `spdk_tgt` app * create a custom NVMe-oF command to implement the seek_data and seek_hole functionalities of bdev used by `spdk_dd` to skip holes With the second solution we could export the volume via NVMe-oF to a dedicated node where to perform the backup with `spdk_dd`application. The restore operation can be done in a couple of way: * read the backup sparse file and write its content into the longhorn block device. In this way data will be fully replicated * clone from backup over each replica, importing the backup sparse file into a new thinly provisioned logical volume. We can perform this operation over the local node, owner of the new volume, if for the backup process we choose to develop a JSON-RPC to export/import logical volume to/from sparse files. Otherwise we can do it or over a dedicated node with `spdk_dd` application, that handle sparse file with SEEK_HOLE and SEEK_DATA functionalities of `lseek`. If we leverage the same backup & restore mechanism of `longhorn-engine`, we can restore a backup done by the actual engine to a SPDK volume. ### Remote Control The JSON-RPC API by default is only available over the `/var/tmp/spdk.sock` Unix domain socket, but SPDK offer the sample python script [rpc_http_proxy](https://spdk.io/doc/jsonrpc_proxy.html) that provides http server which listens for JSON objects from users. Otherwise we could use the `socat` application to forward requests received from an IP socket towards a Unix socket. Both `socat` and `rpc_http_proxy` can perform user authentication with password. ### Upgrade Strategy What kind of upgrade/migration will we support? For out-of-cluster migration we can use the Restore procedure to create SPDK logical volumes starting from existing Longhorn files. Instead for in-cluster migration we can retain read support for the old format, writing new data over SPDK. Whatabout `spdk_tgt` updates, we can perform a rolling update strategy updating nodes one by one. Stopping `spdk_tgt` over a node will cause: * stop of all the volumes controlled in the node. To avoid service interruption the node must be evacuated before the update. The cheat is to delay the update until the node has to be rebooted for a kernel update. * stop of all the replicas hosted in the node. This is not a problem because during the update the I/O will be redirected towards other replica of the volume. To make a clean update of a node, before to stop `spdk_tgt`, we have to notify all the nodes that have a bdev imported via NMVMe-oF from this node to detach controllers involved. Moreover this is a good time to introduce backup versioning, which allows us to change/improve the backup format [REF: GH3175](https://github.com/longhorn/longhorn/issues/3175) ### Future Work * For Edge use cases, energy efficiency is important. We may need further enhancements and an interrupt-driven mode during low load periods for the scheduler. [Here](https://www.snia.org/educational-library/spdk-schedulers-%E2%80%93-saving-cpu-cores-polled-mode-storage-application-2021) an introduction to SPDK Schedulers that describes briefly the interrupt mode. ### Roadmap For Longhorn 1.5, we need to have the below capabilities: * replica (RAID1) * snapshot (create, delete/purge, revert) * replica rebuilding * volume clone For 1.6, we need the rest of the feature parity functions: * volume backup & restore * DR volume restore (incremental restore from another volume backup) * volume encryption * create volume from the backing image * create backing image from volume   * volume expansion * volume trim * volume metrics (bandwidth, latency, IOPS) * volume data integrity (snapshot checksum) SPDK uses a quarterly release cycle, next release will be 23.01 (January 2023). Assuming actual RAID1 implementation will be available in 23.01 release, actually the JSON-RPC we need to implement over SPDK are: * suspend I/O operation * copy a snapshot over an arbitrary bdev * add bdev to raid1 in read balancing disabled mode * enable/disable bdev in raid1 read balancing * export/import file to/from bdev or implement seek_data/hole in NVMe-oF The first development is necessary for snapshot, the last one for backup/restore and the other three developments are necessary for replica rebuilding. The snapshot copy has already been discussed with SPDK core maintainers, so an upstream development can be made. ### Limitations Actual RAID1 implementation is not still complete, so actually we have some limitations: * read balancing has been developed but is still under review, so it is available only in SPDK Gerrit * replica rebuild is still under development, so it isn't available. As a consequence of this, actually RAID1 miss the functionality to add a new base bdev to an existing RAID1 bdev. ================================================ FILE: enhancements/20230103-recurring-snapshot-cleanup.md ================================================ # Recurring Snapshot Cleanup ## Summary Currently, Longhorn's recurring job automatically cleans up older snapshots of volumes to retain no more than the defined snapshot number. However, this is limited to the snapshot created by the recurring job. For the non-recurring volume snapshots or snapshots created by backups, the user needs to clean them manually. Having periodic snapshot cleanup could help to delete/purge those extra snapshots regardless of the creation method. ### Related Issues https://github.com/longhorn/longhorn/issues/3836 ## Motivation ### Goals Introduce new recurring job types: - `snapshot-delete`: periodically remove and purge all kinds of snapshots that exceed the retention count. - `snapshot-cleanup`: periodically purge removable or system snapshots. ### Non-goals [optional] `None` ## Proposal - Introduce two new `RecurringJobType`: - snapshot-delete - snapshot-cleanup - Recurring job periodically deletes and purges the snapshots for RecurringJob using the `snapshot-delete` task type. Longhorn will retain snapshots based on the given retain number. - Recurring job periodically purges the snapshots for RecurringJob using the `snapshot-cleanup` task type. ### User Stories - The user can create a RecurringJob with `spec.task=snapshot-delete` to instruct Longhorn periodically delete and purge snapshots. - The user can create a RecurringJob with `spec.task=snapshot-cleanup` to instruct Longhorn periodically purge removable or system snapshots. ### User Experience In Detail #### Recurring Snapshot Deletion 1. Have some volume backups and snapshots. 1. Create RecurringJob with the `snapshot-delete` task type. ```yaml apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-snap-delete-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} name: recurring-snap-delete-per-min retain: 2 task: snapshot-delete ``` 1. Assign the RecurringJob to volume. 1. Longhorn deletes all expired snapshots. As a result of the above example, the user will see two snapshots after the job completes. #### Recurring Snapshot Cleanup 1. Have some system snapshots. 1. Create RecurringJob with the `snapshot-cleanup` task type. ```yaml apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-snap-cleanup-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} name: recurring-snap-cleanup-per-min task: snapshot-cleanup ``` 1. Assign the RecurringJob to volume. 1. Longhorn deletes all expired system snapshots. As a result of the above example, the user will see 0 system snapshot after the job completes. ### API changes `None` ## Design ### Implementation Overview #### The RecurringJob `snapshot-delete` Task Type 1. List all expired snapshots (similar to the current `listSnapshotNamesForCleanup` implementation), and use as the [cleanupSnapshotNames](https://github.com/longhorn/longhorn-manager/blob/d20e1ca6e04b229b9823c1a941d865929007874c/app/recurring_job.go#L418) in `doSnapshotCleanup`. 1. Continue with the current implementation to purge snapshots. #### The RecurringJob `snapshot-cleanup` Task Type 1. Do snapshot purge only in `doSnapshotCleanup`. ### RecurringJob Mutate 1. Mutate the `Recurringjob.Spec.Retain` to 0 when the task type is `snapshot-cleanup` since retain value has no effect on the purge. ### Test plan #### Test Recurring Snapshot Delete 1. Create volume. 1. Create 2 volume backups. 1. Create 2 volume snapshots. 1. Create a snapshot RecurringJob with the `snapshot-delete` task type. 1. Assign the RecurringJob to volume. 1. Wait until the recurring job is completed. 1. Should see the number of snapshots matching the Recurring job `spec.retain`. #### Test Recurring Snapshot Cleanup 1. Create volume. 1. Create 2 volume system snapshots, ex: delete replica, online expansion. 1. Create a snapshot RecurringJob with the `snapshot-cleanup` task type. 1. Assign the RecurringJob to volume. 1. Wait until the recurring job is completed. 1. Should see the volume has 0 system snapshots. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230108-improve-backup-and-restore-efficiency-using-multiple-threads-and-compression-methods.md ================================================ # Improve Backup and Restore Efficiency using Multiple Threads and Faster Compression Methods ## Summary Longhorn is capable of backing up or restoring volume in multiple threads and using more efficient compression methods for improving Recovery Time Objective (RTO). ### Related Issues [https://github.com/longhorn/longhorn/issues/5189](https://github.com/longhorn/longhorn/issues/5189) ## Motivation ### Goals - Support multi-threaded volume backup and restore. - Support efficient compression algorithm (`lz4`) and disable compression. - Support backward compatibility of existing backups compressed by `gzip`. ### Non-goals - Larger backup block size helps improve the backup efficiency more and decrease the block lookup operations. In the enhancement, the adaptive large backup block size is not supported and will be handled in https://github.com/longhorn/longhorn/issues/5215. ## Proposal 1. Introduce multi-threaded volume backup and restore. Number of backup and restore threads are configurable by uses. 2. Introduce efficient compression methods. By default, the compression method is `lz4`, and user can globally change it to `none` or `gzip`. Additionally, the per-volume compression method can be customized. 3. Existing backups compressed by `gzip` will not be impacted. ## User Stories Longhorn supports the backup and restore of volumes. Although the underlying computing and storage are powerful, the single thread implementation and low efficiency `gzip` compression method lead to slower backup and restore times and poor RTO.The enhancement aims to increase backup and restore efficiency through the use of multiple threads and efficient compression methods. The new parameters can be configured to accommodate a variety of applications and platforms, such as limiting the number of threads in an edge device or disabling compression for multimedia data. ### User Experience In Details - For existing volumes that already have backups, the compression method remains `gzip` for backward compatibility. Multi-threaded backups and restores are supported for subsequent backups. - By default, the global backup compression method is set to `lz4`. By editing the global setting `backup-compression-method`, users can configure the compression method to `none` or `gzip`. The backup compression method can be customized per volume by editing `volume.spec.backupCompressionMethod` for different data format in the volume. - Number of backup threads per backup is configurable by the global setting `backup-concurrent-limit`. - Number of restore threads per backup is configurable by the global setting `restore-concurrent-limit`. - Changing the compression method of a volume having backups is not supported. ### CLI Changes - Add `compression-method` to longhorn-engine binary `backup create` command. - Add `concurrent-limit` to longhorn-engine binary `backup create` command. - Add `concurrent-limit` to longhorn0engine binary `backup restore` command. ### API Changes - engine-proxy - Add `compressionMethod` and `concurrentLimit` to EngineSnapshotBackup method. - Add `concurrentLimit` to `EngineBackupRestore` method. - syncagent - Add `compressionMethod` and `concurrentLimit` to syncagent `BackupCreate` method. - Add `concurrentLimit` to syncagent `BackupRestore` method. ## Design ### Implementation Overview #### Global Settings - backup-compression-method - This setting allows users to specify backup compression method. - Options: - `none`: Disable the compression method. Suitable for multimedia data such as encoded images and videos. - `lz4`: Suitable for text files. - `gzip`: A bit of higher compression ratio but relatively slow. Not recommended. - Default: lz4 - backup-concurrent-limit - This setting controls how many worker threads per backup job concurrently. - Default: 5 - restore-concurrent-limit - This setting controls how many worker threads per restore job concurrently. - Default: 5 #### CRDs 1. Introduce `volume.spec.backupCompressionMethod` 2. Introduce `backup.status.compressionMethod` #### Backup A producer-consumer pattern is used to achieve multi-threaded backups. In this implementation, there are one producer and multiple consumers which is controlled by the global setting `backup-concurrent-limit`. - Producer - Open the disk file to be backed up and create a `Block` channel. - Iterate the blocks in the disk file. - Skip sparse blocks. - Send the data blocks information including offset and size to `Block` channel. - Close the `Block` channel after finishing the iteration. - Consumers - Block handling goroutines (consumers) are created and consume blocks from the `Block` channel. - Processing blocks - Calculate the checksum of the incoming block. - Check the in-memory `processingBlocks` map to determine whether the block is being processed. - If YES, end up appending the block to `Blocks` that record the blocks processed in the backup. - If NO, check the remote backupstore to determine whether the block exists. - If YES, append the block to `Blocks`. - If NO, compress the block, upload the block, and end up appending it to the `Blocks`. - After the blocks have been consumed and the `Block` channel has been closed, the goroutines are terminated. Then, update the volume and backup metadata files in remote backupstore. #### Restore A producer-consumer pattern is used to achieve multi-threaded restores. In this implementation, there are one producer and multiple consumers which is controlled by the global setting `restore-concurrent-limit`. - Producer - Create a `Block` channel. - Open the backup metadata file and get the information, offset, size and checksum, of the blocks. - Iterate the blocks and send the block information to the `Block` channel. - Close the `Block` channel after finishing the iteration. - Consumers - Block handling goroutines (consumers) are created and consume blocks from the `Block` channel. - It is necessary for each consumer to open the disk file in order to avoid race conditions between the seek and write operations. - Read the block data from the backupstore, verify the data integrity and write to the disk file. - After the blocks have been consumed and the `Block` channel has been closed, the goroutines are terminated. ### Performance Benchmark In summary, the backup throughput is increased by 15X when using `lz4` and `10` concurrent threads in comparison with the backup in Longhorn v1.4.0. The restore (to a volume with 3 replica) throughput is increased by 140%, and the throughput is limited by the IO bound of the backupstore server. #### Setup | | | |---|---| | Platform | Equinix | | Host | Japan-Tokyo/m3.small.x86 | | CPU | Intel(R) Xeon(R) E-2378G CPU @ 2.80GHz | | RAM | 64 GiB | | Disk | Micron_5300_MTFD | | OS | Ubuntu 22.04.1 LTS(kernel 5.15.0-53-generic) | | Kubernetes | v1.23.6+rke2r2 | | Longhorn | master-branch + backup improvement | | Nodes | 3 nodes | | Backupstore target | external MinIO S3 (m3.small.x86) | | Volume | 50 GiB containing 1GB filesystem metadata and 10 GiB random data (3 replicas) | #### Results - Single-Threaded Backup and Restore by Different Compression Methods ![Single-Threaded Backup and Restore by Different Compression Methods](image/backup_perf/compression-methods.png) - Multi-Threaded Backup ![Multi-Threaded Backup](image/backup_perf/multi-threaded-backup.png) - Multi-Threaded Restore to One Volume with 3 Replicas ![Multi-Threaded Restore to One Volume with 3 Replicas](image/backup_perf/multi-threaded-restore-to-volume-3-replicas.png) Restore hit the IO bound of the backupstore server, because the throughput is saturated from 5 worker threads. - Multi-Threaded Restore to One Volume with 1 Replica ![Multi-Threaded Restore to One Volume with 1 Replica](image/backup_perf/multi-threaded-restore-to-volume-1-replica.png) ## Test Plan ### Integration Tests 1. Create a volumes and then create backups using the compression method, `none`, `lz4` or `gzip` and different number of backup threads. The backups should succeed. 2. Restore the backups created in step 1 by different number of restore threads. Verify the data integrity of the disk files. ================================================ FILE: enhancements/20230116-smb-cifs-backup-store-support.md ================================================ # SMB/CIFS Backup Store Support ## Summary Longhorn supports SMB/CIFS share as a backup storage. ### Related Issues https://github.com/longhorn/longhorn/issues/3599 ## Motivation ### Goals - Support SMB/CIFS share as a backup storage. ## Proposal - Introduce SMB/CIFS client for supporting SMB/CIFS as a backup storage. ## User Stories Longhorn already supports NFSv4 and S3 servers as backup storage. However, certain users may encounter compatibility issues with their backup servers, particularly those running on Windows, as the protocols for NFSv4 and S3 are not always supported. To address this issue, the enhancement will enhance support for backup storage options with a focus on the commonly used SMB/CIFS protocol, which is compatible with both Linux and Windows-based servers. ### User Experience In Details - Check each Longhorn node's kernel supports the CIFS filesystem by ``` cat /boot/config-`uname -r` | grep CONFIG_CIFS ``` - Install the CIFS filesystem user-space tools `cifs-utils` on each Longhorn node - Users can configure a SMB/CIFS share as a backup storage - Set **Backup Target**. The path to a SMB/CIFS share is like ```bash cifs://${IP address}/${share name} ``` - Set **Backup Target Credential Secret** - Create a secret and deploy it ```yaml apiVersion: v1 kind: Secret metadata: name: cifs-secret namespace: longhorn-system type: Opaque data: CIFS_USERNAME: ${CIFS_USERNAME} CIFS_PASSWORD: ${CIFS_PASSWORD} ``` - Set the setting **Backup Target Credential Secret** to `cifs-secret` ## Design ### Implementation Overview - longhorn-manager - Introduce the fields `CIFS_USERNAME` and `CIFS_PASSWORD` in credentials. The two fields are passed to engine and replica processes for volume backup and restore operations. - backupstore - Implement SMB/CIFS register/unregister and mount/unmount functions ### Test Plan ### Integration Tests 1. Set a SMB/CIFS share as backup storage. 2. Back up volumes to the backup storage and the operation should succeed. 3. Restore backups and the operation should succeed. ================================================ FILE: enhancements/20230303-consolidate-instance-managers.md ================================================ # Consolidate Longhorn Instance Managers ## Summary Longhorn architecture includes engine and replica instance manager pods on each node. After the upgrade, Longhorn adds an additional engine and replica instance manager pods. When the cluster is set with a default request of 12% guaranteed CPU, all instance manager pods will occupy 12% * 4 CPUs per node. Nevertheless, this caused high base resource requirements and is likely unnecessary. ``` NAME STATE E-CPU(CORES) E-MEM(BYTES) R-CPU(CORES) R-MEM(BYTES) CREATED-WORKLOADS DURATION(MINUTES) AGE demo-0 (no-IO) Complete 8.88m 24Mi 1.55m 43Mi 5 10 22h demo-0-bs-512b-5g Complete 109.70m 66Mi 36.46m 54Mi 5 10 16h demo-0-bs-1m-10g Complete 113.16m 65Mi 36.63m 56Mi 5 10 14h demo-0-bs-5m-10g Complete 114.17m 64Mi 31.37m 54Mi 5 10 42m ``` Aiming to simplify the architecture and free up some resource requests, this document proposes to consolidate the engine and replica instance managers into a single pod. This consolidation will not affect any data plane operations or volume migration. As the engine process is the primary consumer of CPU resources, merging the instance managers will result in a 50% reduction in CPU requests for instance managers. This is because there will only be one instance manager pod for both process types. ### Related Issues Phase 1: - https://github.com/longhorn/longhorn/issues/5208 Phase 2: - https://github.com/longhorn/longhorn/issues/5842 - https://github.com/longhorn/longhorn/issues/5844 ## Motivation ### Goals - Having single instance manager pods to run replica and engine processes. - After the Longhorn upgrade, the previous engine instance manager should continue to handle data plane operations for attached volumes until they are detached. And the replica instance managers should continue servicing data plane operations until the volume engine is upgraded or volume is detached. - Automatically clean up any engine/replica instance managers when all instances (process) get removed. - Online/offline upgrade volume engine should be functional. The replicas will automatically migrate to use the new `aio` (all-in-one) type instance managers, and the `engine` type instance manager will continue to serve until the first volume detachment. - The Pod Disruption Budget (PDB) handling for cluster auto-scaler and node drain should work as expected. ### Non-goals [optional] `None` ## Proposal To ensure uninterrupted upgrades, this enhancement will be implemented in two phases. The existing `engine`/`replica` instance manager may coexist with the consolidated instance manager during the transition. Phase 1: - Introduce a new `aio` instance manager type. The `engine` and `replica` instance manager types will be deprecated and continue to serve for the upgraded volumes until the first volume detachment. - Introduce new `Guaranteed Instance Manager CPU` setting, `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` settings will be deprecated and continues to serve for the upgraded volumes until the first volume detachment. Phase 2: - Remove all instance manager types. - Remove the `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` settings. ### User Stories - For freshly installed Longhorn, the user will see `aio` type instance managers. - For upgraded Longhorn with all volume detached, the user will see the `engine`, and `replica` instance managers removed and replaced by `aio` type instance managers. - For upgraded Longhorn with volume attached, the user will see existing `engine`, and `replica` instance managers still servicing the old attached volumes and the new `aio` type instance manager servicing new volume attachments. ### User Experience In Detail #### New Installation 1. User creates and attaches a volume. ``` > kubectl -n longhorn-system get volume NAME STATE ROBUSTNESS SCHEDULED SIZE NODE AGE demo-0 attached unknown 21474836480 ip-10-0-1-113 12s > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 124m instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 124m instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 124m > kubectl -n longhorn-system get lhim/instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc -o yaml apiVersion: longhorn.io/v1beta2 kind: InstanceManager metadata: creationTimestamp: "2023-03-16T10:48:59Z" generation: 1 labels: longhorn.io/component: instance-manager longhorn.io/instance-manager-image: imi-8d41c3a4 longhorn.io/instance-manager-type: aio longhorn.io/managed-by: longhorn-manager longhorn.io/node: ip-10-0-1-113 name: instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc namespace: longhorn-system ownerReferences: - apiVersion: longhorn.io/v1beta2 blockOwnerDeletion: true kind: Node name: ip-10-0-1-113 uid: 00c0734b-f061-4b28-8071-62596274cb18 resourceVersion: "926067" uid: a869def6-1077-4363-8b64-6863097c1e26 spec: engineImage: "" image: c3y1huang/research:175-lh-im nodeID: ip-10-0-1-113 type: aio status: apiMinVersion: 1 apiVersion: 3 currentState: running instanceEngines: demo-0-e-06d4c77d: spec: name: demo-0-e-06d4c77d status: endpoint: "" errorMsg: "" listen: "" portEnd: 10015 portStart: 10015 resourceVersion: 0 state: running type: engine instanceReplicas: demo-0-r-ca78cab4: spec: name: demo-0-r-ca78cab4 status: endpoint: "" errorMsg: "" listen: "" portEnd: 10014 portStart: 10000 resourceVersion: 0 state: running type: replica ip: 10.42.0.238 ownerID: ip-10-0-1-113 proxyApiMinVersion: 1 proxyApiVersion: 4 ``` - The engine and replica instances(processes) created in the `aio` type instance manager. #### Upgrade With Volumes Detached 1. User has a Longhorn v1.4.0 cluster and a volume in the detached state. ``` > kubectl -n longhorn-system get volume NAME STATE ROBUSTNESS SCHEDULED SIZE NODE AGE demo-1 detached unknown 21474836480 12s > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-r-1278a39fa6e6d8f49eba156b81ac1f59 running replica ip-10-0-1-113 3m44s instance-manager-e-1278a39fa6e6d8f49eba156b81ac1f59 running engine ip-10-0-1-113 3m44s instance-manager-e-45ad195db7f55ed0a2dd1ea5f19c5edf running engine ip-10-0-1-105 3m41s instance-manager-r-45ad195db7f55ed0a2dd1ea5f19c5edf running replica ip-10-0-1-105 3m41s instance-manager-e-225a2c7411a666c8eab99484ab632359 running engine ip-10-0-1-102 3m42s instance-manager-r-225a2c7411a666c8eab99484ab632359 running replica ip-10-0-1-102 3m42s ``` 1. User upgraded Longhorn to v1.5.0. ``` > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 112s instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 48s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 47s ``` - Unused `engine` type instance managers removed. - Unused `replica` type instance managers removed. - 3 `aio` type instance managers created. 1. User upgraded volume engine. 1. User attaches the volume. ``` > kubectl -n longhorn-system get volume NAME STATE ROBUSTNESS SCHEDULED SIZE NODE AGE demo-1 attached healthy 21474836480 ip-10-0-1-113 4m51s > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 3m58s instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 2m54s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 2m53s > kubectl -n longhorn-system get lhim/instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc -o yaml apiVersion: longhorn.io/v1beta2 kind: InstanceManager metadata: creationTimestamp: "2023-03-16T13:03:15Z" generation: 1 labels: longhorn.io/component: instance-manager longhorn.io/instance-manager-image: imi-8d41c3a4 longhorn.io/instance-manager-type: aio longhorn.io/managed-by: longhorn-manager longhorn.io/node: ip-10-0-1-113 name: instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc namespace: longhorn-system ownerReferences: - apiVersion: longhorn.io/v1beta2 blockOwnerDeletion: true kind: Node name: ip-10-0-1-113 uid: 12eb73cd-e9de-4c45-875d-3eff7cfb1034 resourceVersion: "3762" uid: c996a89a-f841-4841-b69d-4218ed8d8c6e spec: engineImage: "" image: c3y1huang/research:175-lh-im nodeID: ip-10-0-1-113 type: aio status: apiMinVersion: 1 apiVersion: 3 currentState: running instanceEngines: demo-1-e-b7d28fb3: spec: name: demo-1-e-b7d28fb3 status: endpoint: "" errorMsg: "" listen: "" portEnd: 10015 portStart: 10015 resourceVersion: 0 state: running type: engine instanceReplicas: demo-1-r-189c1bbb: spec: name: demo-1-r-189c1bbb status: endpoint: "" errorMsg: "" listen: "" portEnd: 10014 portStart: 10000 resourceVersion: 0 state: running type: replica ip: 10.42.0.28 ownerID: ip-10-0-1-113 proxyApiMinVersion: 1 proxyApiVersion: 4 ``` - The engine and replica instances(processes) created in the `aio` type instance manager. #### Upgrade With Volumes Attached 1. User has a Longhorn v1.4.0 cluster and a volume in the attached state. ``` > kubectl -n longhorn-system get volume NAME STATE ROBUSTNESS SCHEDULED SIZE NODE AGE demo-2 attached healthy 21474836480 ip-10-0-1-113 35s > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-r-1278a39fa6e6d8f49eba156b81ac1f59 running replica ip-10-0-1-113 2m41s instance-manager-r-45ad195db7f55ed0a2dd1ea5f19c5edf running replica ip-10-0-1-105 119s instance-manager-r-225a2c7411a666c8eab99484ab632359 running replica ip-10-0-1-102 119s instance-manager-e-1278a39fa6e6d8f49eba156b81ac1f59 running engine ip-10-0-1-113 2m41s instance-manager-e-225a2c7411a666c8eab99484ab632359 running engine ip-10-0-1-102 119s instance-manager-e-45ad195db7f55ed0a2dd1ea5f19c5edf running engine ip-10-0-1-105 119s ``` 1. User upgraded Longhorn to v1.5.0. ``` > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-r-1278a39fa6e6d8f49eba156b81ac1f59 running replica ip-10-0-1-113 5m24s instance-manager-r-45ad195db7f55ed0a2dd1ea5f19c5edf running replica ip-10-0-1-105 4m42s instance-manager-r-225a2c7411a666c8eab99484ab632359 running replica ip-10-0-1-102 4m42s instance-manager-e-1278a39fa6e6d8f49eba156b81ac1f59 running engine ip-10-0-1-113 5m24s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 117s instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 33s instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 32s ``` - 2 unused `engine` type instance managers removed. - 3 `aio` type instance managers created. 1. User upgraded online volume engine. ``` > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 6m53s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 8m18s instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 6m54s instance-manager-e-1278a39fa6e6d8f49eba156b81ac1f59 running engine ip-10-0-1-113 11m ``` - All `replica` type instance manager migrated to `aio` type instance managers. 1. User detached the volume. ``` > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 8m38s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 10m instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 8m39s ``` - The `engine` type instance managers removed. 1. User attached the volume. ``` > kubectl -n longhorn-system get volume NAME STATE ROBUSTNESS SCHEDULED SIZE NODE AGE demo-2 attached healthy 21474836480 ip-10-0-1-113 12m > kubectl -n longhorn-system get lhim NAME STATE TYPE NODE AGE instance-manager-7e59c9f2ef7649630344050a8d5be68e running aio ip-10-0-1-102 9m40s instance-manager-8f81ca7c3bf95bbbf656be6ac2d1b7c4 running aio ip-10-0-1-105 9m39s instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc running aio ip-10-0-1-113 11m > kubectl -n longhorn-system get lhim/instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc -o yaml apiVersion: longhorn.io/v1beta2 kind: InstanceManager metadata: creationTimestamp: "2023-03-16T13:12:41Z" generation: 1 labels: longhorn.io/component: instance-manager longhorn.io/instance-manager-image: imi-8d41c3a4 longhorn.io/instance-manager-type: aio longhorn.io/managed-by: longhorn-manager longhorn.io/node: ip-10-0-1-113 name: instance-manager-b34d5db1fe1e2d52bcfb308be3166cfc namespace: longhorn-system ownerReferences: - apiVersion: longhorn.io/v1beta2 blockOwnerDeletion: true kind: Node name: ip-10-0-1-113 uid: 6d109c40-abe3-42ed-8e40-f76cfc33e4c2 resourceVersion: "4339" uid: 01556f2c-fbb4-4a15-a778-c73df518b070 spec: engineImage: "" image: c3y1huang/research:175-lh-im nodeID: ip-10-0-1-113 type: aio status: apiMinVersion: 1 apiVersion: 3 currentState: running instanceEngines: demo-2-e-65845267: spec: name: demo-2-e-65845267 status: endpoint: "" errorMsg: "" listen: "" portEnd: 10015 portStart: 10015 resourceVersion: 0 state: running type: engine instanceReplicas: demo-2-r-a2bd415f: spec: name: demo-2-r-a2bd415f status: endpoint: "" errorMsg: "" listen: "" portEnd: 10014 portStart: 10000 resourceVersion: 0 state: running type: replica ip: 10.42.0.31 ownerID: ip-10-0-1-113 proxyApiMinVersion: 1 proxyApiVersion: 4 ``` - The engine and replica instances(processes) created in the `aio` type instance manager. ### API changes - Introduce new `instanceManagerCPURequest` in `Node` resource. - Introduce new `instanceEngines` in InstanceManager resource. - Introduce new `instanceReplicas` in InstanceManager resource. ## Design ### Phase 1: All-in-one Instance Manager Implementation Overview Introducing a new instance manager type to have Longhorn continue to service existing attached volumes for Longhorn v1.5.x. #### New Instance Manager Type - Introduce a new `aio` (all-in-one) instance manager type to differentiate the handling of the old `engine`/`replica` instance managers and the new consolidated instance managers. - When getting InstanceManagers by instance of the attached volume, retrieve the InstanceManager from the instance manager list using the new `aio` type. #### InstanceManager `instances` Field Replacement For New InstanceManagers - New InstanceManagers will use the `instanceEngines` and `instanceReplicas` fields, replacing the `instances` field. - For the existing InstanceManagers for the attached Volumes, the `instances` field will remain in use. #### Instance Manager Execution - Rename the `engine-manager` script to `instance-manager`. - Bump up version to `4`. #### New Instance Manager Pod - Replace `engine` and `replica` pod creation with spec to use for `aio` instance manager pod. ``` > kubectl -n longhorn-system get pod/instance-manager-0d96990c6881c828251c534eb31bfa85 -o yaml apiVersion: v1 kind: Pod metadata: annotations: longhorn.io/last-applied-tolerations: '[]' creationTimestamp: "2023-03-01T08:13:03Z" labels: longhorn.io/component: instance-manager longhorn.io/instance-manager-image: imi-a1873aa3 longhorn.io/instance-manager-type: aio longhorn.io/managed-by: longhorn-manager longhorn.io/node: ip-10-0-1-113 name: instance-manager-0d96990c6881c828251c534eb31bfa85 namespace: longhorn-system ownerReferences: - apiVersion: longhorn.io/v1beta2 blockOwnerDeletion: true controller: true kind: InstanceManager name: instance-manager-0d96990c6881c828251c534eb31bfa85 uid: 51c13e4f-d0a2-445d-b98b-80cca7080c78 resourceVersion: "12133" uid: 81397cca-d9e9-48f6-8813-e7f2e2cd4617 spec: containers: - args: - instance-manager - --debug - daemon - --listen - 0.0.0.0:8500 env: - name: TLS_DIR value: /tls-files/ image: c3y1huang/research:174-lh-im imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 3 periodSeconds: 5 successThreshold: 1 tcpSocket: port: 8500 timeoutSeconds: 4 name: instance-manager resources: requests: cpu: 960m securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /host mountPropagation: HostToContainer name: host - mountPath: /engine-binaries/ mountPropagation: HostToContainer name: engine-binaries - mountPath: /host/var/lib/longhorn/unix-domain-socket/ name: unix-domain-socket - mountPath: /tls-files/ name: longhorn-grpc-tls - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access-hkbfc readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true nodeName: ip-10-0-1-113 preemptionPolicy: PreemptLowerPriority priority: 0 restartPolicy: Never schedulerName: default-scheduler securityContext: {} serviceAccount: longhorn-service-account serviceAccountName: longhorn-service-account terminationGracePeriodSeconds: 30 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 volumes: - hostPath: path: / type: "" name: host - hostPath: path: /var/lib/longhorn/engine-binaries/ type: "" name: engine-binaries - hostPath: path: /var/lib/longhorn/unix-domain-socket/ type: "" name: unix-domain-socket - name: longhorn-grpc-tls secret: defaultMode: 420 optional: true secretName: longhorn-grpc-tls - name: kube-api-access-hkbfc projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3607 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2023-03-01T08:13:03Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2023-03-01T08:13:04Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2023-03-01T08:13:04Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2023-03-01T08:13:03Z" status: "True" type: PodScheduled containerStatuses: - containerID: containerd://cb249b97d128e47a7f13326b76496656d407fd16fc44b5f1a37384689d0fa900 image: docker.io/c3y1huang/research:174-lh-im imageID: docker.io/c3y1huang/research@sha256:1f4e86b92b3f437596f9792cd42a1bb59d1eace4196139dc030b549340af2e68 lastState: {} name: instance-manager ready: true restartCount: 0 started: true state: running: startedAt: "2023-03-01T08:13:03Z" hostIP: 10.0.1.113 phase: Running podIP: 10.42.0.27 podIPs: - ip: 10.42.0.27 qosClass: Burstable startTime: "2023-03-01T08:13:03Z" ``` #### Controllers Change - Map the status of the engine/replica process to the corresponding instanceEngines/instanceReplicas fields in the InstanceManager instead of the instances field. To ensure backward compatibility, the instances field will continue to be utilized by the pre-upgrade attached volume. - Ensure support for the previous version's attached volumes with the old engine/replica instance manager types. - Replace the old engine/replica InstanceManagers with the aio type instance manager during replenishment. #### New Setting - Introduce a new `Guaranteed Instance Manager CPU` setting for the new `aio` instance manager pod. - The `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` will co-exist with this setting in Longhorn v1.5.x. ### Phase 2 - Deprecations Overview Based on the assumption when upgrading from v1.5.x to 1.6.x, volumes should have detached at least once and migrated to `aio` type instance managers. Then the cluster should not have volume depending on `engine` and `replica` type instance managers. Therefore in this phase, remove the related types and settings. #### Old Instance Manager Types - Remove the `engine`, `replica`, and `aio` instance manager types. There is no need for differentiation. ### Old Settings - Remove the `Guaranteed Engine Manager CPU` and `Guaranteed Replica Manager CPU` settings. The settings have already been replaced by the `Guaranteed Instance Manager CPU` setting in phase 1. #### Controllers Change - Remove support for engine/replica InstanceManager types. ### Test plan Support new `aio` instance manager type and run regression test cases. ### Upgrade strategy The `instances` field in the instance manager custom resource will still be utilized by old instance managers of the attached volume. ## Note [optional] `None` ================================================ FILE: enhancements/20230307-pdb-for-longhon-csi-and-webhook.md ================================================ # Use PDB to protect Longhorn components from drains ## Summary Some Longhorn components should be available to correctly handle cleanup/detach Longhorn volumes during the draining process. They are: `csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, `longhorn-conversion-webhook`, `share-manager`, `instance-manager`, and daemonset pods in `longhorn-system` namespace. This LEP outlines our existing solutions to protect these components, the issues of these solutions, and the proposal for improvement. ### Related Issues https://github.com/longhorn/longhorn/issues/3304 ## Motivation ### Goals 1. Have better ways to protect Longhorn components (`csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, `longhorn-conversion-webhook`) without demanding the users to specify the draining flags to skip these pods. ## Proposal 1. Our existing solutions to protect these components are: * For `instance-manager`: dynamically create/delete instance manager PDB * For Daemonset pods in `longhorn-system` namespace: we advise the users to specify `--ignore-daemonsets` to ignore them in the `kubectl drain` command. This actually follows the [best practice](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#:~:text=If%20there%20are%20pods%20managed%20by%20a%20DaemonSet%2C%20you%20will%20need%20to%20specify%20%2D%2Dignore%2Ddaemonsets%20with%20kubectl%20to%20successfully%20drain%20the%20node) * For `csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, and `longhorn-conversion-webhook`: we advise the user to specify `--pod-selector` to ignore these pods 1. Proposal for `csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, and `longhorn-conversion-webhook`:
The problem with the existing solution is that sometime, users could not specify `--pod-selector` for the `kubectl drain` command. For example, for the users that are using the project [System Upgrade Controller](https://github.com/rancher/system-upgrade-controller), they don't have option to specify `--pod-selector`. Also, we would like to have a more automatic way instead of relying on the user to set kubectl drain options. Therefore, we propose the following design: * Longhorn manager automatically create PDBs for `csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, and `longhorn-conversion-webhook` with `minAvailable` set to 1. This will make sure that each of these deployment has at least 1 running pod during the draining process. * Longhorn manager continuously watches the volumes and removes the PDBs once there is no attached volume. This should work for both single-node and multi-node cluster. ### User Stories #### Story 1 Before the enhancement, users would need to specify the drain options for drain command to exclude Longhorn pods. Sometimes, this is not possible when users use third-party solution to drain and upgrade kubernetes, such as System Upgrade Controller. #### Story 2 ### User Experience In Detail After the enhancement, the user can doesn't need to specify the drain options for the drain command to exclude Longhorn pods. ### API changes None ## Design ### Implementation Overview Create a new controller inside Longhorn manager called `longhorn-pdb-controller`, the controller listens for the changes for `csi-attacher`, `csi-provisioner`, `longhorn-admission-webhook`, `longhorn-conversion-webhook`, and Longhorn volumes to adjust the PDB correspondingly. ### Test plan https://github.com/longhorn/longhorn/issues/3304#issuecomment-1467174481 ### Upgrade strategy No Upgrade is needed ## Note In the original Github ticket, we mentioned that we need to add PDB to protect share manager pod from being drained before its workload pods because if share manager pod doesn't exist then its volume cannot be unmounted in the CSI flow. However, with the fix https://github.com/longhorn/longhorn/issues/5296, we can always umounted the volume even if the share manager is not running. Therefore, we don't need to protect share manager pod. ================================================ FILE: enhancements/20230309-recurring-filesystem-trim.md ================================================ # Recurring Filesystem Trim ## Summary Longhorn currently supports the [filesystem trim](./20221103-filesystem-trim.md) feature, which allows users to reclaim volume disk spaces of deleted files. However, this is a manual process, which can be time-consuming and inconvenient. To improve user experience, Longhorn could automate the process by implementing a new RecurringJob `filesystem-trim` type. This enhancement enables regularly freeing up unused volume spaces and reducing the need for manual interventions. ### Related Issues https://github.com/longhorn/longhorn/issues/5186 ## Motivation ### Goals Introduce a new recurring job type called `filesystem-trim` to periodically trim the volume filesystem to reclaim disk spaces. ### Non-goals [optional] `None` ## Proposal To extend the RecurringJob custom resource definition by adding new `RecurringJobType: filesystem-trim`. ### User Stories To schedule regular volume filesystem trims, user can create a RecurringJob with `spec.task=filesystem-trim` and associating it with volumes. ### User Experience In Detail #### Recurring Filesystem Trim 1. The user sees workload volume size has increased over time. 1. Create RecurringJob with the `filesystem-trim` task type and assign it to the volume. ```yaml apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-fs-trim-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} name: recurring-fs-trim-per-min retain: 0 task: filesystem-trim ``` 1. The RecurringJob runs and relaims some volume spaces. ### API changes `None` ## Design ### Implementation Overview #### The RecurringJob `filesystem-trim` Task Type 1. Call Volume API `ActionTrimFilesystem` when the RecurringJob type is `filesystem-trim`. ### RecurringJob Mutate 1. Mutate the `Recurringjob.Spec.Retain` to 0 when the task type is `filesystem-trim` as it is not effective for this type of task. ### Test plan #### Test Recurring Filesystem Trim 1. Create workload. 1. Create a file with some data in the workload. 1. Volume actual size should increase. 1. Delete the file. 1. Volume actual size should not decrease. 1. Create RecurringJob with type `filesystem-trim` and assign to the workload volume. 1. Wait for RecurringJob to complete. 1. Volume actual size should decrease. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230315-upgrade-path-enforcement.md ================================================ # Upgrade Path Enforcement ## Summary Currently, Longhorn does not enforce the upgrade path, even though we claim Longhorn only supports upgrading from the previous stable release, for example, upgrading to 1.5.x is only supported from 1.4.x or 1.5.0. Without upgrade enforcement, we will allow users to upgrade from any previous version. This will cause extra testing efforts to cover all upgrade paths. Additionally, the goal of this enhancement is to support rollback after upgrade failure and prevent downgrades. ### Related Issues https://github.com/longhorn/longhorn/issues/5131 ## Motivation ### Goals - Enforce an upgrade path to prevent users from upgrading from any unsupported version. After rejecting the user's upgrade attempt, the user's Longhorn setup should remain intact without any impacts. - Upgrade Longhorn from the authorized versions to a major release version. - Support rollback the failed upgrade to the previous version. - Prevent unexpected downgrade. ### Non-goals - Automatic rollback if the upgrade failed. ## Proposal - When upgrading with `kubectl`, it will check the upgrade path at entry point of the pods for `longhorn-manager`, `longhorn-admission-webhook`, `longhorn-conversion-webhook` and `longhorn-recovery-backend`. - When upgrading with `Helm` or as a `Rancher App Marketplace`, it will check the upgrade path by a `pre-upgrade` job of `Helm hook` ### User Stories - As the admin, I want to upgrade Longhorn from x.y.* or x.(y+1).0 to x.(y+1).* by `kubectl`, `Helm` or `Rancher App Marketplace`, so that the upgrade should succeed. - As the admin, I want to upgrade Longhorn from the previous authorized versions to a new major/minor version by `kubectl`, `Helm`, or `Rancher App Marketplace`, so that the upgrade should succeed. - As the admin, I want to upgrade Longhorn from x.(y-1).* to x.(y+1).* by 'kubectl', 'Helm' or 'Rancher App Marketplace', so that the upgrade should be prevented and the system with the current version continues running w/o any interruptions. - As the admin, I want to roll back Longhorn from the failed upgrade to the previous install by `kubectl`, `Helm`, or `Rancher App Marketplace`, so that the rollback should succeed. - As the admin, I want to downgrade Longhorn to any lower version by `kubectl`, `Helm`, or `Rancher App Marketplace`, so that the downgrade should be prevented and the system with the current version continues running w/o any interruptions. ### User Experience In Detail #### Upgrade Longhorn From x.y.* or x.(y+1).0 To x.(y+1).* ##### Upgrade With `kubectl` 1. Install Longhorn on any Kubernetes cluster by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.y.*/deploy/longhorn.yaml ``` or ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.(y+1).0/deploy/longhorn.yaml ``` 1. After Longhorn works normally, upgrade Longhorn by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.(y+1).*/deploy/longhorn.yaml ``` 1. It will be allowed and Longhorn will be upgraded successfully. ##### Upgrade With `Helm` Or `Rancher App Marketplace` 1. Install Longhorn x.y.* or x.(y+1).0 with Helm as [Longhorn Install with Helm document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-helm/) or install Longhorn x.y.* or x.(y+1).0 with a Rancher Apps as [Longhorn Install as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-rancher/) 1. Upgrade to Longhorn x.(y+1).* with Helm as [Longhorn Upgrade with Helm document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-with-helm) or upgrade to Longhorn x.(y+1).* with a Rancher Catalog App as [Longhorn Upgrade as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-as-a-rancher-catalog-app) 1. It will be allowed and Longhorn will be upgraded successfully. #### Upgrade Longhorn From The Authorized Versions To A Major Release Version ##### Upgrade With `kubectl` 1. Install Longhorn on any Kubernetes cluster by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.y.*/deploy/longhorn.yaml ``` 1. After Longhorn works normally, upgrade Longhorn by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v(x+1).0.*/deploy/longhorn.yaml ``` 1. It will be allowed and Longhorn will be upgraded successfully. ##### Upgrade With `Helm` Or `Rancher App Marketplace` 1. Install Longhorn x.y.* with Helm such as [Longhorn Install with Helm document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-helm/) or install Longhorn x.y.* with a Rancher Apps as [Longhorn Install as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-rancher/) 1. Upgrade to Longhorn (x+1).0.* with Helm as [Longhorn Upgrade with Helm document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-with-helm) or upgrade to Longhorn (x+1).0.* with a Rancher Catalog App as [Longhorn Upgrade as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-as-a-rancher-catalog-app) 1. It will be allowed and Longhorn will be upgraded successfully. #### Upgrade Longhorn From x.(y-1).* To x.(y+1).* ##### Upgrade With `kubectl` 1. Install Longhorn on any Kubernetes cluster by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.(y-1).*/deploy/longhorn.yaml ``` 1. After Longhorn works normally, upgrade Longhorn by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.(y+1).*/deploy/longhorn.yaml ``` 1. It will be not allowed and Longhorn will block the upgrade for `longhorn-manager`, `longhorn-admission-webhook`, `longhorn-conversion-webhook` and `longhorn-recovery-backend`. 1. Users need to roll back Longhorn manually to restart `longhorn-manager` pods. ##### Upgrade With `Helm` Or `Rancher App Marketplace` 1. Install Longhorn x.(y-1).* with Helm as [Longhorn Install with Helm document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-helm/) or install Longhorn x.(y-1).* with a Rancher Apps as [Longhorn Install as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-rancher/) 1. Upgrade to Longhorn x.(y+1).* with Helm as [Longhorn Upgrade with Helm document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-with-helm) or upgrade to Longhorn x.(y+1).* with a Rancher Catalog App as [Longhorn Upgrade as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-as-a-rancher-catalog-app) 1. It will not be allowed and a `pre-upgrade`job of `Helm hook` failed makes the whole helm upgrading process failed. 1. Longhorn is intact and continues serving. #### Roll Back Longhorn From The Failed Upgrade To The Previous Install ##### Roll Back With `kubectl` 1. Users need to recover Longhorn by using this command again: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/[previous installed version]/deploy/longhorn.yaml ``` 1. Longhorn will be rolled back successfully. 1. And users might need to delete new components introduced by new version Longhorn manually. ##### Roll Back With `Helm` Or `Rancher App Marketplace` 1. Users need to recover Longhorn with `Helm` by using commands: ```shell helm history longhorn # to get previous installed Longhorn REVISION helm rollback longhorn [REVISION] ``` or ```shell helm upgrade longhorn longhorn/longhorn --namespace longhorn-system --version [previous installed version] ``` 1. Users need to recover Longhorn with `Rancher Catalog Apps` by upgrading the previous installed Longhorn version at `Rancher App Marketplace` again. 1. Longhorn will be rolled back successfully. ##### Manually Cleanup Example When users try to upgrade Longhorn from v1.3.x to v1.5.x, a new deployment `longhorn-recovery-backend` will be introduced and the upgrade will fail. Users need to delete the deployment `longhorn-recovery-backend` manually after rolling back Longhorn #### Downgrade Longhorn To Any Lower Version ##### Downgrade With `kubectl` 1. Install Longhorn on any Kubernetes cluster by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.y.*/deploy/longhorn.yaml ``` 1. After Longhorn works normally, upgrade Longhorn by using this command: ```shell kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/vx.(y-z).*/deploy/longhorn.yaml ``` 1. It will be not allowed and Longhorn will block the downgrade for `longhorn-manager`. [or `longhorn-admission-webhook`, `longhorn-conversion-webhook` and `longhorn-recovery-backend` if downgrading version had these components] 1. Users need to roll back Longhorn manually to restart `longhorn-manager` pods. ##### Downgrade With `Helm` Or `Rancher App Marketplace` 1. Install Longhorn x.y.* with Helm as [Longhorn Install with Helm document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-helm/) or install Longhorn x.y.* with a Rancher Apps as [Longhorn Install as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/install/install-with-rancher/) 1. Downgrade to Longhorn (x-z).y.* or x.(y-z).* with Helm as [Longhorn Upgrade with Helm document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-with-helm) or downgrade to Longhorn (x-z).y.* or x.(y-z).* with a Rancher Catalog App as [Longhorn Upgrade as a Rancher Apps & Marketplace document](https://longhorn.io/docs/1.4.1/deploy/upgrade/longhorn-manager/#upgrade-as-a-rancher-catalog-app) 1. It will not be allowed and a `pre-upgrade`job of `Helm hook` failed makes the whole helm downgrading process failed. 1. Longhorn is intact and continues serving. ### API changes `None` ## Design ### Implementation Overview #### Blocking Upgrade With `kubectl` Check the upgrade path is supported or not at entry point of the `longhorn-manager`, `longhorn-admission-webhook`, `longhorn-conversion-webhook` and `longhorn-recovery-backend` 1. Get Longhorn current version `currentVersion` by the function `GetCurrentLonghornVersion` 1. Get Longhorn upgrading version `upgradeVersion` from `meta.Version` 1. Compare currentVersion and upgradeVersion, only allow authorized version upgrade (e.g., 1.3.x to 1.5.x is not allowed) as following table. | currentVersion | upgradeVersion | Allow | | :-: | :-: | :-: | | x.y.* | x.(y+1).* | ✓ | | x.y.0 | x.y.* | ✓ | | x.y.* | (x+1).y.* | ✓ | | x.(y-1).* | x.(y+1).* | X | | x.(y-2).* | x.(y+1).* | X | | x.y.* | x.(y-1).* | X | | x.y.* | x.y.(*-1) | X | 1. Downgrade is not allowed. 2. When the upgrade path is not supported, new created pods of the `longhorn-manager`, `longhorn-admission-webhook`, `longhorn-conversion-webhook` and `longhorn-recovery-backend` will show logs and broadcast events for the upgrade path is not supported and return errors. 3. Previous installed Longhorn will work normally still. #### Blocking Upgrade With `Helm` Or `Rancher App Marketplace` 1. Add a new job for pre-upgrade hook of `Helm` as the [`post-upgrade` job](https://github.com/longhorn/longhorn/blob/master/chart/templates/postupgrade-job.yaml). ```txt apiVersion: batch/v1 kind: Job metadata: annotations: "helm.sh/hook": pre-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed name: longhorn-pre-upgrade ... spec: ... template: metadata: name: longhorn-pre-upgrade ... spec: containers: - name: longhorn-post-upgrade ... command: - longhorn-manager - pre-upgrade env: ... ``` 1. When upgrading starts, the `pre-upgrade` job will start to run firstly and it will be failed if the upgrade path is not supported then `Helm` upgrading process will be failed. ### Test plan #### Test Supported Upgrade Path 1. Install Longhorn v1.4.x. 1. Wait for all pods ready. 1. Create a Volume and write some data. 1. Upgrade to Longhorn v1.5.0. 1. Wait for all pods upgraded successfully. 1. Check if data is not corrupted. #### Test Unsupported Upgrade Path 1. Install Longhorn v1.3.x. 1. Wait for all pods ready. 1. Create a Volume and write some data. 1. Upgrade to Longhorn v1.5.0. 1. Upgrading process will be stuck or failed. 1. Check if data is not corrupted. 1. Rollback to Longhorn v1.3.x with the same setting. 1. Longhorn v1.3.x will work normally. ### Upgrade strategy `None` ## Note `None` ================================================ FILE: enhancements/20230417-extend-csi-snapshot-to-support-backingimage.md ================================================ # Title Extend CSI snapshot to support Longhorn BackingImage ## Summary In Longhorn, we have BackingImage for VM usage. We would like to extend the CSI Snapshotter to support BackingImage management. ### Related Issues [BackingImage Management via VolumeSnapshot #5005](https://github.com/longhorn/longhorn/issues/5005) ## Motivation ### Goals Extend the CSI snapshotter to support: - Create Longhorn BackingImage - Delete Longhorn BackingImage - Creating a new PVC from CSI snapshot that is associated with a Longhorn BackingImage ### Non-goals [optional] - Can support COW over each relative base image for delta data transfer for better space efficiency. (Will be in next improvement) - User can backup a BackingImage based volume and restore it in another cluster without manually preparing BackingImage in a new cluster. ## Proposal ### User Story With this improvement, users can use standard CSI VolumeSnapshot as the unified interface for BackingImage creation, deletion and restoration of a Volume. ### User Experience In Detail To use this feature, users need to deploy the CSI snapshot CRDs and related Controller 1. The instructions are already on our document: https://longhorn.io/docs/1.4.1/snapshots-and-backups/csi-snapshot-support/enable-csi-snapshot-support/ 2. Create a VolumeSnapshotClass with type `bi` which refers to BackingImage ```yaml kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1 metadata: name: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete parameters: type: bi export-type: qcow2 # default to raw if it is not provided ``` #### BackingImage creation via VolumenSnapshot resource Users can create a BackingImage of a Volume by creation of VolumeSnapshot. Example below for a Volume named `test-vol` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: persistentVolumeClaimName: test-vol ``` Longhorn will create a BackingImage **exported** from this Volume. #### Restoration via VolumeSnapshot resource Users can create a volume based on a prior created VolumeSnapshot. Example below for a Volume named `test-vol-restore` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-vol-restore spec: storageClassName: longhorn dataSource: name: test-snapshot-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` Longhorn will create a Volume based on the BackingImage associated with the VolumeSnapshot. #### Restoration of an existing BackingImage (pre-provisioned) Users can request the creation of a Volume based on a prior BackingImage which was not created via the CSI VolumeSnapshot. With the BackingImage already existing, users need to create the VolumeSnapshotContent with an associated VolumeSnapshot. The `snapshotHandle` of the VolumeSnapshotContent needs to point to an existing BackingImage. Example below for a Volume named `test-restore-existing-backing` and an existing BackingImage `test-bi` - For pre-provisioning, users need to provide following query parameters: - `backingImageDataSourceType`: `sourceType` of existing BackingImage, e.g. `export-from-volume`, `download` - `backingImage`: Name of the BackingImage - you should also provide the `sourceParameters` of existing BackingImage in the `snapshotHandle` for validation. - `export-from-volume`: you should provide - `volume-name` - `export-type` - `download`: you should proviide - `url` - `checksum`: optional ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-existing-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: # NOTE: change this to point to an existing BackingImage in Longhorn snapshotHandle: bi://backing?backingImageDataSourceType=export-from-volume&backingImage=test-bi&volume-name=vol-export-src&export-type=qcow2 volumeSnapshotRef: name: test-snapshot-existing-backing namespace: default ``` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-existing-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: volumeSnapshotContentName: test-existing-backing ``` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-existing-backing spec: storageClassName: longhorn dataSource: name: test-snapshot-existing-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` Longhorn will create a Volume based on the BackingImage associated with the VolumeSnapshot and the VolumeSnapshotContent. #### Restoration of a non-existing BackingImage (on-demand provision) Users can request the creation of a Volume based on a BackingImage which was not created yet with following 2 kinds of data sources. 1. `download`: Download a file from a URL as a BackingImage. 2. `export-from-volume`: Export an existing in-cluster volume as a backing image. Users need to create the VolumeSnapshotContent with an associated VolumeSnapshot. The `snapshotHandle` of the VolumeSnapshotContent needs to provide the parameters for the data source. Example below for a volume named `test-on-demand-backing` and an non-existing BackingImage `test-bi` with two different data sources. 1. `download`: Users need to provide following parameters - `backingImageDataSourceType`: `download` for on-demand download. - `backingImage`: Name of the BackingImage - `url`: The file from a URL as a BackingImage. - `backingImageChecksum`: Optional. Used for checking the checksum of the file. - example yaml: ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: # NOTE: change this to provide the correct parameters snapshotHandle: bi://backing?backingImageDataSourceType=download&backingImage=test-bi&url=https%3A%2F%2Flonghorn-backing-image.s3-us-west-1.amazonaws.com%2Fparrot.qcow2&backingImageChecksum=bd79ab9e6d45abf4f3f0adf552a868074dd235c4698ce7258d521160e0ad79ffe555b94e7d4007add6e1a25f4526885eb25c53ce38f7d344dd4925b9f2cb5d3b volumeSnapshotRef: name: test-snapshot-on-demand-backing namespace: default ``` 2. `export-from-volume`: Users need to provide following parameters - `backingImageDataSourceType`: `export-form-volume` for on-demand export. - `backingImage`: Name of the BackingImage - `volume-name`: Volume to be exported for the BackingImage - `export-type`: Currently Longhorn supports `raw` or `qcow2` - example yaml: ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: # NOTE: change this to provide the correct parameters snapshotHandle: bi://backing?backingImageDataSourceType=export-from-volume&backingImage=test-bi&volume-name=vol-export-src&export-type=qcow2 volumeSnapshotRef: name: test-snapshot-on-demand-backing namespace: default ``` Users then can create corresponding VolumeSnapshot and PVC ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: # NOTE: change this to point to the prior VolumeSnapshotContent volumeSnapshotContentName: test-on-demand-backing ``` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-on-demand-backing spec: storageClassName: longhorn dataSource: name: test-snapshot-on-demand-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` ### API changes No changes necessary ## Design ### Implementation Overview We add a new type `bi` to the parameter `type` in the VolumeSnapshotClass. It means that the CSI VolumeSnapshot created with this VolumeSnapshotClass is associated with a Longhorn BackingImage. #### CreateSnapshot function When the users create VolumeSnapshot and the volumeSnapshotClass `type` is `bi` ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: persistentVolumeClaimName: test-vol ``` We do: - Get the name of the Volume - The name of the BackingImage will be same as the VolumeSnapshot `test-snapshot-backing`. - Check if a BackingImage with the same name as the requested VolumeSnapshot already exists. Return success without creating a new BackingImage. - Create a BackingImage. - Get `export-type` from VolumeSnapshotClass parameter `export-type`, default to `raw.` - Encode the `snapshotId` as `bi://backing?backingImageDataSourceType=export-from-volume&backingImage=test-snapshot-backing&volume-name=${VolumeName}&export-type=raw` - This `snapshotId` will be used in the later CSI CreateVolume and DeleteSnapshot call. #### CreateVolume function - If VolumeSource type is `VolumeContentSource_Snapshot`, decode the `snapshotId` to get the parameters. - `bi://backing?backingImageDataSourceType=${TYPE}&backingImage=${BACKINGIMAGE_NAME}&backingImageChecksum=${backingImageChecksum}&${OTHER_PARAMETERS}` - If BackingImage with the given name already exists, create the volume. - If BackingImage with the given name does not exists, we prepare it first. There are 2 kinds of types which are `export-from-volume` and `download`. - For `download`, it means we have to prepare the BackingImage before creating the Volume. We first decode other parameters from `snapshotId` and create the BackingImage. - For `export-from-volume`, it means we have to prepare the BackingImage before creating the Volume. We first decode other parameters from `snapshotId` and create the BackingImage. NOTE: we already have related code for preparing the BackingImage with type `download` or `export-from-volume` before creating a Volume, [here](https://github.com/longhorn/longhorn-manager/blob/master/csi/controller_server.go#L195) #### DeleteSnapshot function - Decode the `snapshotId` to get the name of the BackingImage. Then we delete the BackingImage directly. ### Test plan Integration test plan. #### Prerequisite 1. Deploy the csi snapshot CRDs, Controller as instructed at https://longhorn.io/docs/1.4.1/snapshots-and-backups/csi-snapshot-support/enable-csi-snapshot-support/ 2. Create a VolumeSnapshotClass with type `bi` ```yaml # Use v1 as an example kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1 metadata: name: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete parameters: type: bi ``` #### Scenarios 1: Create VolumeSnapshot from a Volume - Success 1. Create a Volume `test-vol` of 5GB. Create PV/PVC for the Volume. 2. Create a workload using the Volume. Write some data to the Volume. 3. Create a VolumeSnapshot with following yaml: ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: persistentVolumeClaimName: test-vol ``` 4. Verify that BacingImage is created. - Verify the properties of BackingImage - `sourceType` is `export-from-volume` - `volume-name` is `test-vol` - `export-type` is `raw` 5. Delete the VolumeSnapshot `test-snapshot-backing` 6. Verify the BacingImage is deleted #### Scenarios 2: Create new Volume from CSI snapshot 1. Create a Volume `test-vol` of 5GB. Create PV/PVC for the Volume. 2. Create a workload using the Volume. Write some data to the Volume. 3. Create a VolumeSnapshot with following yaml: ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: persistentVolumeClaimName: test-vol ``` 4. Verify that BacingImage is created. 5. Create a new PVC with following yaml: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-pvc spec: storageClassName: longhorn dataSource: name: test-snapshot-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` 5. Attach the PVC `test-restore-pvc` to a workload and verify the data 6. Delete the PVC #### Scenarios 3: Restore pre-provisioned BackingImage 1. Create a BackingImage `test-bi` using longhorn test raw image `https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.qcow2` 2. Create a VolumeSnapshotContent with `snapshotHandle` pointing to BackingImage `test-bi` and provide the parameters. ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-existing-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: snapshotHandle: bi://backing?backingImageDataSourceType=download&backingImage=test-bi&url=https%3A%2F%2Flonghorn-backing-image.s3-us-west-1.amazonaws.com%2Fparrot.qcow2&backingImageChecksum=bd79ab9e6d45abf4f3f0adf552a868074dd235c4698ce7258d521160e0ad79ffe555b94e7d4007add6e1a25f4526885eb25c53ce38f7d344dd4925b9f2cb5d3b volumeSnapshotRef: name: test-snapshot-existing-backing namespace: default ``` 3. Create a VolumeSnapshot associated with the VolumeSnapshotContent ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-existing-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: volumeSnapshotContentName: test-existing-backing ``` 4. Create a PVC with the following yaml ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-existing-backing spec: storageClassName: longhorn dataSource: name: test-snapshot-existing-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` 5. Attach the PVC `test-restore-existing-backing` to a workload and verify the data #### Scenarios 4: Restore on-demand provisioning BackingImage - Type `download` 1. Create a VolumeSnapshotContent with `snapshotHandle` providing the required parameters and BackingImage name `test-bi` ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: snapshotHandle: bi://backing?backingImageDataSourceType=download&backingImage=test-bi&url=https%3A%2F%2Flonghorn-backing-image.s3-us-west-1.amazonaws.com%2Fparrot.qcow2&backingImageChecksum=bd79ab9e6d45abf4f3f0adf552a868074dd235c4698ce7258d521160e0ad79ffe555b94e7d4007add6e1a25f4526885eb25c53ce38f7d344dd4925b9f2cb5d3b volumeSnapshotRef: name: test-snapshot-on-demand-backing namespace: default ``` 2. Create a VolumeSnapshot associated with the VolumeSnapshotContent ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: volumeSnapshotContentName: test-on-demand-backing ``` 3. Create a PVC with the following yaml ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-on-demand-backing spec: storageClassName: longhorn dataSource: name: test-snapshot-on-demand-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` 4. Verify BackingImage `test-bi` is created 5. Attach the PVC `test-restore-on-demand-backing` to a workload and verify the data - Type `export-from-volume` - Success 1. Create a Volme `test-vol` and write some data to it. 2. Create a VolumeSnapshotContent with `snapshotHandle` providing the required parameters and BackingImage name `test-bi` ```yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc driver: driver.longhorn.io deletionPolicy: Delete source: snapshotHandle: bi://backing?backingImageDataSourceType=export-from-volume&backingImage=test-bi&volume-name=test-vol&export-type=qcow2 volumeSnapshotRef: name: test-snapshot-on-demand-backing namespace: default ``` 2. Create a VolumeSnapshot associated with the VolumeSnapshotContent ```yaml apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: test-snapshot-on-demand-backing spec: volumeSnapshotClassName: longhorn-snapshot-vsc source: volumeSnapshotContentName: test-on-demand-backing ``` 3. Create a PVC with the following yaml ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-on-demand-backing spec: storageClassName: longhorn dataSource: name: test-snapshot-on-demand-backing kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 5Gi ``` 4. Verify BackingImage `test-bi` is created 5. Attach the PVC `test-restore-on-demand-backing` to a workload and verify the data ### Upgrade strategy No upgrade strategy needed ## Note [optional] We need to update the docs and examples to reflect the new type of parameter `type` in the VolumeSnapshotClass. ================================================ FILE: enhancements/20230418-azure-blob-storage-backup-store-support.md ================================================ # Azure Blob Storage Backup Store Support ## Summary Longhorn supports Azure Blob Storage as a backup storage. ### Related Issues https://github.com/longhorn/longhorn/issues/1309 ## Motivation ### Goals - Support Azure Blob Storage as a backup storage. ## Proposal - Introduce Azure Blob Storage client for supporting Azure Blob Storage as a backup storage. ## User Stories Longhorn already supports NFSv4, CIFS and S3 servers as backup storage. However, certain users may still want to be able to utilize Azure blob storage to push/pull backups to/from. ### User Experience In Details - Users can configure a Azure Blob Storage as a backup storage - Set **Backup Target**. The path to a Azure Blob Storage is like ```bash azblob://${container}@blob.core.windows.net/${path name} ``` - Set **Backup Target Credential Secret** - Create a secret and deploy it ```yaml apiVersion: v1 kind: Secret metadata: name: azblob-secret namespace: longhorn-system type: Opaque data: AZBLOB_ACCOUNT_NAME: ${AZBLOB_ACCOUNT_NAME} AZBLOB_ACCOUNT_KEY: ${AZBLOB_ACCOUNT_KEY} ``` - Set the setting **Backup Target Credential Secret** to `azblob-secret` ## Design ### Implementation Overview - longhorn-manager - Introduce the fields `AZBLOB_ACCOUNT_NAME` and `AZBLOB_ACCOUNT_KEY` in credentials. The two fields are passed to engine and replica processes for volume backup and restore operations. - backupstore - Implement Azure Blob Storage register/unregister and basic CRUD functions. ## Test Plan ### Integration Tests 1. Set a Azure Blob Storage as backup storage. 2. Create volumes and write some data. 3. Back up volumes to the backup storage and the operation should succeed. 4. Restore backups and operations should succeed. 5. All data is not corrupted. ================================================ FILE: enhancements/20230420-engine-identity-validation.md ================================================ # Engine Identity Validation ## Summary Longhorn-manager communicates with longhorn-engine's gRPC ControllerService, ReplicaService, and SyncAgentService by sending requests to TCP/IP addresses kept up-to-date by its various controllers. Additionally, the longhorn-engine controller server sends requests to the longhorn-engine replica server's ReplicaService and SyncAgentService using TCP/IP addresses it keeps in memory. These addresses are relatively stable in normal operation. However, during periods of high process turnover (e.g. a node reboot or network event), it is possible for one longhorn-engine component to stop and another longhorn-engine component to start in its place using the same ports. If this happens quickly enough, other components with stale address lists attempting to execute requests against the old component may errantly execute requests against the new component. One harmful effect of this behavior that has been observed is the [expansion of an unintended longhorn-engine replica](https://github.com/longhorn/longhorn/issues/5709). This proposal intends to ensure all gRPC requests to longhorn-engine components are actually served by the intended component. ### Related Issues https://github.com/longhorn/longhorn/issues/5709 ## Motivation ### Goals - Eliminate the potential for negative effects caused by a Longhorn component communicating with an incorrect longhorn-engine component. - Provide effective logging when incorrect communication occurs to aide in fixing TCP/IP address related race conditions. ### Non-goals - Fix race conditions within the Longhorn control plane that lead to attempts to communicate with an incorrect longhorn-engine component. - Refactor the in-memory data structures the longhorn-engine controller server uses to keep track of and initiate communication with replicas. ## Proposal Today, longhorn-manager knows the volume name and instance name of the process it is trying to communicate with, but it only uses the TCP/IP information of each process to initiate communication. Additionally, longhorn-engine components are mostly unaware of the volume name (in the case of longhorn-engine's replica server) and instance name (for both longhorn-engine controller and replica servers) they are associated with. If we provide this information to longhorn-engine processes when we start them and then have longhorn-manager provide it on every communication attempt, we can ensure no accidental communication occurs. 1. Add additional flags to the longhorn-engine CLI that inform controller and replica servers of their associated volume and/or instance name. 1. Use [gRPC client interceptors](https://github.com/grpc/grpc-go/blob/master/examples/features/interceptor/README.md) to automatically inject [gRPC metadata](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) (i.e. headers) containing volume and/or instance name information every time a gRPC request is made by a longhorn-engine client to a longhorn-engine server. 1. Use [gRPC server interceptors](https://github.com/grpc/grpc-go/blob/master/examples/features/interceptor/README.md) to automatically validate the volume and/or instance name information in [gRPC metadata](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) (i.e. headers) every time a gRPC request made by a longhorn-engine client is received by a longhorn-engine server. 1. Reject any request (with an appropriate error code) if the provided information does not match the information a controller or replica server was launched with. 1. Log the rejection at the client and the server, making it easy to identify situations in which incorrect communication occurs. 1. Modify instance-manager's `ProxyEngineService` (both server and client) so that longhorn-manager can provide the necessary information for gRPC metadata injection. 1. Modify longhorn-manager so that is makes proper use of the new `ProxyEngineService` client and launches longhorn-engine controller and replica servers with additional flags. ### User Stories #### Story 1 Before this proposal: As an administrator, after an intentional or unintentional node reboot, I notice one or more of my volumes is degraded and new or existing replicas aren't coming online. In some situations, the UI reports confusing information or one or more of my volumes might be unable to attach at all. Digging through logs, I see errors related to mismatched sizes, and at least one replica does appear to have a larger size reported in `volume.meta` than others. I don't know how to proceed. After this proposal: As an administrator, after an intentional or unintentional node reboot, my volumes work as expected. If I choose to dig through logs, I may see some messages about refused requests to incorrect components, but this doesn't seem to negatively affect anything. #### Story 2 Before this proposal: As a developer, I am aware that it is possible for one Longhorn component to communicate with another, incorrect component, and that this communication can lead to unexpected replica expansion. I want to work to fix this behavior. However, when I look at a support bundle, it is very hard to catch this communication occurring. I have to trace TCP/IP addresses through logs, and if no negative effects are caused, I may never notice it. After this proposal: Any time one Longhorn component attempts to communicate with another, incorrect component, it is clearly represented in the logs. ### User Experience In Detail See the user stories above. This enhancement is intended to be largely transparent to the user. It should eliminate rare failures so that users can't run into them. ### API Changes #### Longhorn-Engine Increment the longhorn-engine CLIAPIVersion by one. Do not increment the longhorn-engine CLIAPIMinVersion. The changes in this LEP are backwards compatible. All gRPC metadata validation is by demand of the client. If a less sophisticated (not upgraded) client does not inject any metadata, the server performs no validation. If a less sophisticated (not upgraded) client only injects some metadata (e.g. `volume-name` but not `instance-name`), the server only validates the metadata provided. Add a global `volume-name` flag and a global `engine-instance-name` flag to the engine CLI (e.g. `longhorn -volume-name -engine-instance-name `). Virtually all CLI commands create a controller client and these flags allow appropriate gRPC metadata to be injected into every client request. Requests that reach the wrong longhorn-engine controller server are rejected. Use the global `engine-instance-name` flag and the pre-existing `volume-name` positional argument to allow the longhorn-engine controller server to remember its volume and instance name (e.g. `longhorn -engine-instance-name controller `). Ignore the global `volume-name` flag, as it is redundant. Use the global `volume-name` flag or the pre-existing local `volume-name` flag and a new `replica-instance-name` flag to allow the longhorn-engine replica server to remember its volume and instance name (e.g. `longhorn -volume-name replica -replica-instance-name `). Use the global `volume-name` flag and a new `replica-instance-name` flag to allow the longhorn-engine sync-agent server to remember its volume and instance name (e.g. `longhorn -volume-name sync-agent -replica-instance-name `). Add an additional `replica-instance-name` flag to CLI commands that launch asynchronous tasks that communicate directly with the longhorn-engine replica server (e.g. `longhorn -volume-name add-replica
-size -current-size -replica-instance-name `). All such commands create a replica client and these flags allow appropriate gRPC metadata to be injected into every client request. Requests that reach the wrong longhorn-engine replica server are rejected. Return 9 FAILED_PRECONDITION with an appropriate message when metadata validation fails. This code is chosen in accordance with the [RPC API](https://grpc.github.io/grpc/core/md_doc_statuscodes.html), which instructs developers to use FAILED_PRECONDITION if the client should not retry until the system system has been explicitly fixed. #### Longhorn-Instance-Manager Increment the longhorn-instance-manager InstanceManagerProxyAPIVersion by one. Do not increment the longhorn-instance-manager InstanceManagerProxyAPIMinVersion. The changes in this LEP are backwards compatible. No added fields are required and their omission is ignored. If a less sophisticated (not upgraded) client does not include them, no metadata is injected into engine or replica requests and no validation occurs (the behavior is the same as before the implementation of this LEP). Add `volume_name` and `instance_name` fields to the `ProxyEngineRequest` protocol buffer message. This message, which currently only contains an `address` field, is included in all `ProxyEngineService` RPCs. Updated clients can pass information about the engine process they expect to be communicating with in these fields. When instance-manager creates an asynchronous task to carry out the requested operation, the resulting controller client includes the gRPC interceptor described above. Add `replica_instance_name` fields to any `ProxyEngineService` RPC associated with an asynchronous task that communicates directly with a longhorn-engine replica server. When instance-manager creates the task, the resulting replica client includes the gRPC interceptor described above. Return 5 NOT FOUND with an appropriate message when metadata validation fails at a lower layer. (The particular return code is definitely open to discussion.) ## Design ### Implementation Overview #### Interceptors (longhorn-engine) Add a gRPC server interceptor to all `grpc.NewServer` calls. ```golang server := grpc.NewServer(withIdentityValidationInterceptor(volumeName, instanceName)) ``` Implement the interceptor so that it validates metadata with best effort. ```golang func withIdentityValidationInterceptor(volumeName, instanceName string) grpc.ServerOption { return grpc.UnaryInterceptor(identityValidationInterceptor(volumeName, instanceName)) } func identityValidationInterceptor(volumeName, instanceName string) grpc.UnaryServerInterceptor { // Use a closure to remember the correct volumeName and/or instanceName. return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { md, ok := metadata.FromIncomingContext(ctx) if ok { incomingVolumeName, ok := md["volume-name"] // Only refuse to serve if both client and server provide validation information. if ok && volumeName != "" && incomingVolumeName[0] != volumeName { return nil, status.Errorf(codes.InvalidArgument, "Incorrect volume name; check controller address") } } if ok { incomingInstanceName, ok := md["instance-name"] // Only refuse to serve if both client and server provide validation information. if ok && instanceName != "" && incomingInstanceName[0] != instanceName { return nil, status.Errorf(codes.InvalidArgument, "Incorrect instance name; check controller address") } } // Call the RPC's actual handler. return handler(ctx, req) } } ``` Add a gRPC client interceptor to all `grpc.Dial` calls. ```golang connection, err := grpc.Dial(serviceUrl, grpc.WithInsecure(), withIdentityValidationInterceptor(volumeName, instanceName)) ``` Implement the interceptor so that it injects metadata with best effort. ```golang func withIdentityValidationInterceptor(volumeName, instanceName string) grpc.DialOption { return grpc.WithUnaryInterceptor(identityValidationInterceptor(volumeName, instanceName)) } func identityValidationInterceptor(volumeName, instanceName string) grpc.UnaryClientInterceptor { // Use a closure to remember the correct volumeName and/or instanceName. return func(ctx context.Context, method string, req any, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { if volumeName != "" { ctx = metadata.AppendToOutgoingContext(ctx, "volume-name", volumeName) } if instanceName != "" { ctx = metadata.AppendToOutgoingContext(ctx, "instance-name", instanceName) } return invoker(ctx, method, req, reply, cc, opts...) } } ``` Modify all client constructors to include this additional information. Wherever these client packages are consumed (e.g. the replica client is consumed by the controller, both the replica and the controller clients are consumed by longhorn-manager), callers can inject this additional information into the constructor and get validation for free. ```golang func NewControllerClient(address, volumeName, instanceName string) (*ControllerClient, error) { // Implementation. } ``` #### CLI Commands (longhorn-engine) Add additional flags to all longhorn-engine CLI commands depending on their function. E.g. command that launches a server: ```golang func ReplicaCmd() cli.Command { return cli.Command{ Name: "replica", UsageText: "longhorn controller DIRECTORY SIZE", Flags: []cli.Flag{ // Other flags. cli.StringFlag{ Name: "volume-name", Value: "", Usage: "Name of the volume (for validation purposes)", }, cli.StringFlag{ Name: "replica-instance-name", Value: "", Usage: "Name of the instance (for validation purposes)", }, }, // Rest of implementation. } } ``` E.g. command that directly communicates with both a controller and replica server. ```golang func AddReplicaCmd() cli.Command { return cli.Command{ Name: "add-replica", ShortName: "add", Flags: []cli.Flag{ // Other flags. cli.StringFlag{ Name: "volume-name", Required: false, Usage: "Name of the volume (for validation purposes)", }, cli.StringFlag{ Name: "engine-instance-name", Required: false, Usage: "Name of the controller instance (for validation purposes)", }, cli.StringFlag{ Name: "replica-instance-name", Required: false, Usage: "Name of the replica instance (for validation purposes)", }, }, // Rest of implementation. } } ``` #### Instance-Manager Integration Modify the ProxyEngineService server functions so that they can make correct use of the changes in longhorn-engine. Funnel information from the additional fields in the ProxyEngineRequest message and in appropriate ProxyEngineService RPCs into the longhorn-engine task and controller client constructors so it can be used for validation. ```protobuf message ProxyEngineRequest{ string address = 1; string volume_name = 2; string instance_name = 3; } ``` Modify the ProxyEngineService client functions so that consumers can provide the information required to enable validation. #### Longhorn-Manager Integration Ensure the engine and replica controllers launch engine and replica processes with `-volume-name` and `-engine-instance-name` or `-replica-instance-name` flags so that these processes can validate identifying gRPC metadata coming from requests. Ensure the engine controller supplies correct information to the ProxyEngineService client functions so that identity validation can occur in the lower layers. #### Example Validation Flow This issue/LEP was inspired by [longhorn/longhorn#5709](https://github.com/longhorn/longhorn/issues/5709). In the situation described in this issue: 1. An engine controller with out-of-date information (including a replica address the associated volume does not own) [issues a ReplicaAdd command](https://github.com/longhorn/longhorn-manager/blob/a7dd20cdbdb1a3cea4eb7490f14d94d2b0ef273a/controller/engine_controller.go#L1819) to instance-manager's EngineProxyService. 2. Instance-manager creates a longhorn-engine task and [calls its AddReplica method](https://github.com/longhorn/longhorn-instance-manager/blob/0e0ec6dcff9c0a56a67d51e5691a1d4a4f397f4b/pkg/proxy/replica.go#L35). 3. The task makes appropriate calls to a longhorn-engine controller and replica. The ReplicaService's [ExpandReplica command](https://github.com/longhorn/longhorn-engine/blob/1f57dd9a235c6022d82c5631782020e84da22643/pkg/sync/sync.go#L509) is used to expand the replica before a followup failure to actually add the replica to the controller's backend. After this improvement, the above scenario will be impossible: 1. Both the engine and replica controllers will launch engine and replica processes with the `-volume-name` and `-engine-instance-name` or `replica-instance-name` flags. 2. When the engine controller issues a ReplicaAdd command, it will do so using the expanded embedded `ProxyEngineRequest` message (with `volume_name` and `instance_name` fields) and an additional `replica_instance_name` field. 3. Instance-manager will create a longhorn-engine task that automatically injects `volume-name` and `instance-name` gRPC metadata into each controller request. 4. When the task issues an ExpandReplica command, it will do so using a client that automatically injects `volume-name` and `instance-name` gRPC metadata into it. 5. If either the controller or the replica does not agree with the information provided, gRPC requests will fail immediately and there will be no change in any longhorn-engine component. ### Test plan #### TODO: Integration Test Plan In my test environment, I have experimented with: - Running new versions of all components, making gRPC calls to the longhorn-engine controller and replica processes with wrong gRPC metadata, and verifying that these calls fail. - Running new versions of all components, making gRPC calls to instance-manager with an incorrect volume-name or instance name, and verifying that these calls fail. - Running new versions of all components, adding additional logging to longhorn-engine and verifying that metadata validation is occurring during the normal volume lifecycle. This is really a better fit for a negative testing scenario (do something that would otherwise result in improper communication, then verify that communication fails), but we have already eliminated the only known recreate for [longhorn/longhorn#5709](https://github.com/longhorn/longhorn/issues/5709). #### Engine Integration Test Plan Rework test fixtures so that: - All controller and replica processes are created with the information needed for identity validation. - It is convenient to create controller and replica clients with the information needed for identity validation. - gRPC metadata is automatically injected into controller and replica client requests when clients have the necessary information. Do not modify the behavior of existing tests. Since these tests were using clients with identity validation information, no identity validation is performed. - Modify functions/fixtures that create engine/replica processes to allow the new flags to be passed, but do not pass them by default. - Modify engine/replica clients used by tests to allow for metadata injection, but do not enable it by default. Create new tests that: - Ensure validation fails when a directly created client attempts to communicate with a controller or replica server using the wrong identity validation information. - Ensure validation fails when an indirectly created client (by the engine) tries to communicate with a replica server using the wrong identity validation information. - Ensure validation fails when an indirectly created client (by a CLI command) tries to communicate with a controller or replica server using the wrong identity validation information. ### Upgrade strategy The user will get benefit from this behavior automatically, but only after they have upgraded all associated components to a supporting version (longhorn-manager, longhorn-engine, and CRITICALLY instance-manager). We will only provide volume name and instance name information to longhorn-engine controller and replica processes on a supported version (as governed by the `CLIAPIVersion`). Even if other components are upgraded, when they send gRPC metadata to non-upgraded processes, it will be ignored. We will only populate extra ProxyEngineService fields when longhorn-manager is running with an update ProxyEngineService client. - RPCs from an old client to a new ProxyEngineService server will succeed, but without the extra fields, instance-manager will have no useful gRPC metadata to inject into its longhorn-engine requests. - RPCs from a new client to an old ProxyEngineService will succeed, but instance-manager will ignore the new fields and not inject useful gRPC metadata into its longhorn-engine request. ## Note ### Why gRPC metadata? We initially looked at adding volume name and/or instance name fields to all longhorn-engine ReplicaService and ControllerService calls. However, this would be awkward with some of the existing RPCs. In addition, it doesn't make much intuitive sense. Why should we provide the name of an entity we are communicating with to that entity as part of its API? It makes more sense to think of this identity validation in terms of sessions or authorization/authentication. In HTTP, information of this nature is handled through the use of headers, and metadata is the gRPC equivalent. ### Why gRPC interceptors? We want to ensure the same behavior in every longhorn-engine ControllerService and ReplicaService call so that it is not up to an individual developer writing a new RPC to remember to validate gRPC metadata (and to relearn how it should be done). Interceptors work mostly transparently to ensure identity validation always occurs. ================================================ FILE: enhancements/20230420-upgrade-checker-info-collection.md ================================================ # Upgrade Checker Info Collection The website https://metrics.longhorn.io/ offers valuable insights into how Longhorn is being utilized, which can be accessed by the public. This information serves as a useful reference for user who are new to Longhorn, as well as those considering upgrading Longhorn or the underlying Kubernetes version. Additionally, it is useful for the Longhorn team to understand how it is being used in the real world. To gain a deeper understanding of usage patterns, it would be beneficial to gather additional information on volumes, host systems, and features. This data would not only offer insights into how to further improve Longhorn but also provide valuable ideas on how to steer Longhorn development in the right direction. ## Summary This proposal aims to enhance Longhorn's upgrade checker `extraInfo` by collecting additional information includes node and cluster information, and some Longhorn settings. This proposal introduces a new setting, `Allow Collecting Longhorn Usage Metrics`, to allow users to enable or disable the collection. ### Related Issues https://github.com/longhorn/longhorn/issues/5235 ## Motivation ### Goals 1. Extend collections of user cluster info during upgrade check. 1. Have a new setting to provide user with option to enable or disable the collection. ### Non-goals [optional] `None` ## Proposal 1. Collect and sends through upgrade responder request. - Node info: - Kernel release - OS distro - Disk types (HDD, SSD, NVMe) - Node provider - Cluster info: - Longhorn namespace UID for adaption rate - Number of nodes - Longhorn components CPU and memory usage - Volumes info; such as access mode, frontend, average snapshot per volume, etc. - Some Longhorn settings 1. Introduce new `Allow Collecting Longhorn Usage Metrics` setting. ### User Stories Users can view how Longhorn is being utilized on https://metrics.longhorn.io/. Additionally, users have the ability to disable the collection by Longhorn. ### User Experience In Detail Users can find a list of items that Longhorn collects as extra information in the Longhorn documentation. Users can enable or disable the collection through the `Allow Collecting Longhorn Usage Metrics` setting. This setting can be configured using the UI or through kubectl, similar to other settings. ### API changes `None` ## Design ### Implementation Overview #### `Allow Collecting Longhorn Usage Metrics` Setting - If this value is set to false, extra information will not be collected. - Setting definition: ``` DisplayName: "Allow Collecting Longhorn Usage Metrics" Description: "Enabling this setting will allow Longhorn to provide additional usage metrics to https://metrics.longhorn.io. This information will help us better understand how Longhorn is being used, which will ultimately contribute to future improvements." Category: SettingCategoryGeneral Type: SettingTypeBool Required: true ReadOnly: false Default: "true" ``` #### Extra Info Collection ##### Node Info The following information is sent from each cluster node: - Number of disks of different device (`longhorn_node_disk__count`). > Note: this value may not be accurate if the cluster node is a virtual machine. - Host kernel release (`host_kernel_release`) - Host Os distro (`host_os_distro`) - Kubernetes node provider (`kubernetes_node_provider`) ##### Cluster Info The following information is sent from one of the cluster node: - Longhorn namespace UID (`longhorn_namespace_uid`). - Number of nodes (`longhorn_node_count`). - Number of volumes of different access mode (`longhorn_volume_access_mode__count`). - Number of volumes of different data locality (`longhorn_volume_data_locality__count`). - Number of volumes of different frontend (`longhorn_volume_frontend__count`). - Average volume size (`longhorn_volume_average_size`). - Average volume actual size (`longhorn_volume_average_actual_size`). - Average number of snapshots per volume (`longhorn_volume_average_snapshot_count`). - Average number of replicas per volume (`longhorn_volume_average_number_of_replicas`). - Average Longhorn component CPU usage (`longhorn__average_cpu_usage_core`) - Average Longhorn component CPU usage (`longhorn__average_memory_usage_mib`) - Settings (`longhorn_setting_`): - Settings to exclude: - SettingNameBackupTargetCredentialSecret - SettingNameDefaultEngineImage - SettingNameDefaultInstanceManagerImage - SettingNameDefaultShareManagerImage - SettingNameDefaultBackingImageManagerImage - SettingNameSupportBundleManagerImage - SettingNameCurrentLonghornVersion - SettingNameLatestLonghornVersion - SettingNameStableLonghornVersions - SettingNameDefaultLonghornStaticStorageClass - SettingNameDeletingConfirmationFlag - SettingNameDefaultDataPath - SettingNameUpgradeChecker - SettingNameAllowCollectingLonghornUsage - SettingNameDisableReplicaRebuild (deprecated) - SettingNameGuaranteedEngineCPU (deprecated) - Settings that requires processing to identify their general purpose: - SettingNameBackupTarget (the backup target type/protocol, ex: cifs, nfs, s3) - Settings that should be collected as boolean (true if configured, false if not): - SettingNameTaintToleration - SettingNameSystemManagedComponentsNodeSelector - SettingNameRegistrySecret - SettingNamePriorityClass - SettingNameStorageNetwork - Other settings that should be collected as it is. Example: ``` name: upgrade_request time app_version host_kernel_release host_os_distro kubernetes_node_provider kubernetes_version longhorn_engine_image_average_cpu_usage_core longhorn_engine_image_average_memory_usage_mib longhorn_instance_manager_average_cpu_usage_core longhorn_instance_manager_average_memory_usage_mib longhorn_manager_average_cpu_usage_core longhorn_manager_average_memory_usage_mib longhorn_namespace_uid longhorn_node_count longhorn_node_disk_nvme_count longhorn_setting_allow_node_drain_with_last_healthy_replica longhorn_setting_allow_recurring_job_while_volume_detached longhorn_setting_allow_volume_creation_with_degraded_availability longhorn_setting_auto_cleanup_system_generated_snapshot longhorn_setting_auto_delete_pod_when_volume_detached_unexpectedly longhorn_setting_auto_salvage longhorn_setting_backing_image_cleanup_wait_interval longhorn_setting_backing_image_recovery_wait_interval longhorn_setting_backup_compression_method longhorn_setting_backup_concurrent_limit longhorn_setting_backup_target longhorn_setting_backupstore_poll_interval longhorn_setting_concurrent_automatic_engine_upgrade_per_node_limit longhorn_setting_concurrent_replica_rebuild_per_node_limit longhorn_setting_concurrent_volume_backup_restore_per_node_limit longhorn_setting_crd_api_version longhorn_setting_create_default_disk_labeled_nodes longhorn_setting_default_data_locality longhorn_setting_default_replica_count longhorn_setting_disable_revision_counter longhorn_setting_disable_scheduling_on_cordoned_node longhorn_setting_engine_replica_timeout longhorn_setting_failed_backup_ttl longhorn_setting_fast_replica_rebuild_enabled longhorn_setting_guaranteed_engine_manager_cpu longhorn_setting_guaranteed_instance_manager_cpu longhorn_setting_guaranteed_replica_manager_cpu longhorn_setting_kubernetes_cluster_autoscaler_enabled longhorn_setting_node_down_pod_deletion_policy longhorn_setting_node_drain_policy longhorn_setting_orphan_auto_deletion longhorn_setting_priority_class longhorn_setting_recurring_failed_jobs_history_limit longhorn_setting_recurring_successful_jobs_history_limit longhorn_setting_registry_secret longhorn_setting_remove_snapshots_during_filesystem_trim longhorn_setting_replica_auto_balance longhorn_setting_replica_file_sync_http_client_timeout longhorn_setting_replica_replenishment_wait_interval longhorn_setting_replica_soft_anti_affinity longhorn_setting_replica_zone_soft_anti_affinity longhorn_setting_restore_concurrent_limit longhorn_setting_restore_volume_recurring_jobs longhorn_setting_snapshot_data_integrity longhorn_setting_snapshot_data_integrity_cronjob longhorn_setting_snapshot_data_integrity_immediate_check_after_snapshot_creation longhorn_setting_storage_minimal_available_percentage longhorn_setting_storage_network longhorn_setting_storage_over_provisioning_percentage longhorn_setting_storage_reserved_percentage_for_default_disk longhorn_setting_support_bundle_failed_history_limit longhorn_setting_system_managed_components_node_selector longhorn_setting_system_managed_pods_image_pull_policy longhorn_setting_taint_toleration longhorn_ui_average_cpu_usage_core longhorn_ui_average_memory_usage_mib longhorn_volume_access_mode_rwo_count longhorn_volume_average_actual_size longhorn_volume_average_number_of_replicas longhorn_volume_average_size longhorn_volume_average_snapshot_count longhorn_volume_data_locality_disabled_count longhorn_volume_frontend_blockdev_count value ---- ----------- ------------------- -------------- ------------------------ ------------------ -------------------------------------------- ---------------------------------------------- ------------------------------------------------ -------------------------------------------------- --------------------------------------- ----------------------------------------- ---------------------- ------------------- ----------------------------- ----------------------------------------------------------- ---------------------------------------------------------- ----------------------------------------------------------------- ------------------------------------------------------- ------------------------------------------------------------------ ----------------------------- ---------------------------------------------------- ----------------------------------------------------- ------------------------------------------ ---------------------------------------- ------------------------------ ------------------------------------------ ------------------------------------------------------------------- ---------------------------------------------------------- ---------------------------------------------------------------- -------------------------------- -------------------------------------------------- -------------------------------------- -------------------------------------- ----------------------------------------- ---------------------------------------------------- --------------------------------------- ---------------------------------- --------------------------------------------- ---------------------------------------------- ------------------------------------------------ ----------------------------------------------- ------------------------------------------------------ ---------------------------------------------- ---------------------------------- ------------------------------------- ------------------------------- ---------------------------------------------------- -------------------------------------------------------- -------------------------------- -------------------------------------------------------- ------------------------------------- ------------------------------------------------------ ---------------------------------------------------- ------------------------------------------- ------------------------------------------------ ----------------------------------------- ---------------------------------------------- ---------------------------------------- ------------------------------------------------ -------------------------------------------------------------------------------- ----------------------------------------------------- -------------------------------- ----------------------------------------------------- ------------------------------------------------------------- ---------------------------------------------------- -------------------------------------------------------- ------------------------------------------------------ --------------------------------- ---------------------------------- ------------------------------------ ------------------------------------- ----------------------------------- ------------------------------------------ ---------------------------- -------------------------------------- -------------------------------------------- --------------------------------------- ----- 1683598256887331729 v1.5.0-dev 5.3.18-59.37-default "sles" k3s v1.23.15+k3s1 5m 11 4m 83 22m 85 1b96b299-b785-468b-ab80-b5b5b12fbe00 3 1 false false true true true true 60 300 lz4 5 none 300 0 5 5 longhorn.io/v1beta2 false disabled 3 false true 8 1440 true 12 12 12 false do-nothing block-if-contains-last-replica false false 1 1 false false disabled 30 600 false true 5 false fast-check 0 0 */7 * * false 25 false 200 30 1 false if-not-present false 0 4 3 79816021 2 8589934592 0 3 3 1 1683598257082240493 v1.5.0-dev 5.3.18-59.37-default "sles" k3s v1.23.15+k3s1 1 1 1683598257825718008 v1.5.0-dev 5.3.18-59.37-default "sles" k3s v1.23.15+k3s1 1 1 ``` ### Test plan 1. Set up the upgrade responder server. 1. Verify the database when the `Allow Collecting Longhorn Usage Metrics` setting is enabled or disabled. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230517-set-recurring-job-to-pvc.md ================================================ # Set RecurringJob to PersistentVolumeClaims (PVCs) Managing recurring jobs for Longhorn Volumes is challenging for users utilizing gitops. Primary because gitops operates at the Kubernetes resource level while recurring job labeling is specific to individual Longhorn Volumes. ## Summary This document proposes the implementation of a solution that allows configuring recurring jobs directly on PVCs. By adopting this approach, users will have the capability to manage Volume recurring jobs through the PVCs. ### Related Issues https://github.com/longhorn/longhorn/issues/5791 ## Motivation ### Goals 1. Introduce support for enabling/disabling PVCs as a recurring job label source for the corresponding Volume. 1. The recurring job labels on PVCs are reflected on the associated Volume when the PVC is set as the recurring job label source. ### Non-goals [optional] Sync Volume recurring job labels to PVC. ## Proposal 1. The existing behavior of recurring jobs will remain unchanged, with the Volume's recurring job labeling as the source of truth. 2. When the PVC is enabled as the recurring job label source, its recurring job labels will override all recurring job labels of the associated Volume. ### User Stories As a user, I want to be able to set the RecurringJob label on the PVC. I expect that any updates made to the RecurringJob labels on the PVC will automatically reflect on the associated Volume. ### User Experience In Detail To enable or disable the PVC as the recurring job label source, users can manage it by adding or removing the `recurring-job.longhorn.io/source: enable` label to the PVC. Once the PVC is set as the recurring job label source, any recurring job labels added or removed from the PVC will be automatically synchronized by Longhorn to the associated Volume. ### API changes `None` ## Design ### Implementation Overview #### Sync Volume recurring job labels to PVC #### If the PVC is labeled with `recurring-job.longhorn.io/source: enable`, the volume controller checks and updates the Volume to ensure the recurring job labels stay synchronized with the PVC by detecting recurring job label differences. #### Remove PVC recurring job of the deleting RecurringJob #### As of now, Longhorn includes a feature that automatically removes the Volume recurring job label associated with a deleting RecurringJob. This is also applicable to the PVC. ### Test plan 1. Update PVC recurring job label should reflect on the Volume. 1. Delete RecurringJob custom resource should delete the recurring job labels on both Volume and PVC. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230523-support-spdk-volumes.md ================================================ # Support Volumes using V2 Data Engine ## Summary Longhorn's storage stack, based on iSCSI and a customized protocol, has limitations such as increased I/O latencies and reduced IOPS due to the longer data path. This makes it less suitable for latency-critical applications. To overcome these challenges, Longhorn introduces the Storage Performance Development Kit (SPDK) to enhance overall performance. With SPDK integration, Longhorn optimizes system efficiency, addresses latency concerns, and provides a high-performance storage solution capable of meeting diverse workload demands. ### Related Issues - [[FEATURE] Add a global setting for enabling and disabling SPDK feature](https://github.com/longhorn/longhorn/issues/5778) - [[FEATURE] Support replica scheduling for SPDK volume](https://github.com/longhorn/longhorn/issues/5711) - [[FEATURE] Implement Disk gRPC Service in Instance Manager for collecting SPDK disk statistics from SPDK gRPC service](https://github.com/longhorn/longhorn/issues/5744) - [[FEATURE] Identify and manage orphaned lvols and raid bdevs if the associated Volume resources are not existing](https://github.com/longhorn/longhorn/issues/5827) ## Motivation ### Goals - Introduce backend store drivers - `v1`: legacy data path - `v2`: a newly introduced data path based on SPDK - Introduce disk types and management - Support volume creation, attachment, detachment and deletion - Support orphaned replica collection ### Non-goals [optional] - Support runtime replica rebuilding - Support changing number of replicas of a volume - Support volume expansion - Support volume backup ## Proposal ### User Stories Longhorn's storage stack is built upon iSCSI and a customized protocol. However, the longer data path associated with this architecture introduces certain limitations, resulting in increased I/O latencies and reduced IOPS. Consequently, Longhorn may not be the ideal choice for latency-critical applications, as the performance constraints could impede their deployment on the platform. By incorporating SPDK, Longhorn leverages its capabilities to significantly improve performance levels. The integration of SPDK enables Longhorn to optimize system efficiency, mitigate latency concerns, and deliver a high-performance storage solution that can better meet the demands of diverse workloads. ### User Experience In Detail - Environment Setup - Configure Kernel Modules (uio and uio_pci_generic) and Huge Pages for SPDK ```bash kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/master/deploy/prerequisite/longhorn-spdk-setup.yaml ``` - Install NVMe Userspace Tool and Load `nvme-tcp` Kernel Module nvme-cli on each node and make sure that the version of nvme-cli is equal to or greater than version `1.12` . ```bash kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/master/deploy/prerequisite/longhorn-nvme-cli-installation.yaml ``` - Restart `kubelet` Modifying the Huge Page configuration of a node requires either a restart of kubelet or a complete reboot of the node. This step is crucial to ensure that the changes take effect and are properly applied. - Install Longhorn system - Enable SPDK Support Enable the SPDK feature by changing the `v2-data-engine` setting to `true` after installation. Following this, the instance-manager pods shall be automatically restarted. - Add Disks for volumes using v2 data engine - Legacy disks are classified as `filesystem`-type disks - Add one or multiple `block`-type disks into `node.Spec.Disks` ```bash block-disk-example1: allowScheduling: true evictionRequested: false path: /path/to/block/device storageReserved: 0 tags: [] diskType: block ``` - Create a storage class utilizing the enhanced performance capabilities offered by SPDK ```bash kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-v2-data-engine provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: Immediate parameters: numberOfReplicas: "2" staleReplicaTimeout: "2880" fromBackup: "" fsType: "ext4" dataEngine: "v2" ``` - Create workloads that use Longhorn volumes provisioning based on the storage class. ### API changes ## Design ### Implementation Overview - Global settings - `v2-data-engine`: This setting allows users to enable v2 data engine support. Default: false. - `v2-data-engine-hugepage-limit`: This setting allows users to specify the 2 MiB hugepage size for v2 data engine. Default: 2048. - CRD - Introduce `diskType` in `node.Spec.Disks` - `filesystem`: disks for legacy volumes. These disks, which are actually directories, store and organize data in a hierarchical manner. - `block`: block disks for volumes using v2 data engine The replica scheduler assigns replicas of legacy volumes to `filesystem`-type disks while replicas of volumes using v2 data engine are scheduled to `block`-type disks. - Introduce `backendStoreDriver` in `volume.Spec`, `engine.Spec` and `replica.Spec`. - `backendStoreDriver` is utilized to differentiate between volume types and their associated data paths. - Introduce `Instance`, `Disk` and `SPDK` gRPC services ![gRPC Services](image/spdk_services.png) - `Instance` gRPC service: It is tasked with managing various operations related to instance management, including creation, deletion, retrieval, listing, and watching. An instance, either an engine or a replica of a legacy volume, represents a process. On the other hand, for replicas of volumes using v2 data engine, an instance represents a logical volume. In the case of an engine for an volume using v2 data engine, an instance is associated with a raid bdev, a frontend NVMe target/initiator pair and a bind mount device. - `Disk` gRPC service: It is responsible for managing various disk operations, including creation, deletion, and retrieval. Additionally, it provides functionalities to list or delete replica instances associated with the disks. In the case of a legacy volume, a replica instance is represented as a replica directory on the disk. On the other hand, for an volume using v2 data engine, a replica instance is a replica chained by logical volumes. - `SPDK` gRPC service: It manages replicas chained by logical volumes and engines constructed using SPDK raid1 bdevs. In addition, the service is responsible for the communication with `spdk_tgt`. - Proxy gRPC service APIs - Update gRPC service APIs for support different disk type, filesystem and block, and data engines, v1 and v2. - Disk orchestration Within the Longhorn system, an aio bdev and an lvstore are created on top of a block-type disk. Replicas in terms of logical volumes (lvols) are then created on the lvstore. ![Disks for Volumes Using V2 Data Engine](image/spdk_disks.png) - Orphaned replicas collection The features have been integrated into the existing framework for collecting and cleaning up orphaned replicas. ## Test Plan ## Note [optional] ================================================ FILE: enhancements/20230526-volume-backup-policy-for-longhorn-system-backup.md ================================================ # Volume Backup Policy for Longhorn System Backup The current implementation of the Longhorn system backup lacks integration with the volume backup feature. As a result, users are required to manually ensure that all volume backups are up-to-date before initiating the Longhorn system backup. ## Summary This document proposed to include the volume backup feature in the Longhorn system backup by introducing volume backup policies. By implementing the volume backup policies, users will gain the ability to define how volume data should be backed up during the Longhorn system backup. ### Related Issues https://github.com/longhorn/longhorn/issues/5011 ## Motivation ### Goals 1. **Customization:** By offering different volume backup policy options, users can choose the one best fit with their requirements. 1. **Reduce Manual Efforts:** By integrating volume backup into the Longhorn system backup, users no longer have to ensure that all volume backups are up-to-date before initiating the system backup, 1. **Enhanced Data Integrity:** By aligning the system backup with a new up-to-date volume backups, the restored volume data will be more accurate. Overall, the proposed volume backup policies aim to improve the Longhorn system backup functionality and providing a more robust and customizable system backup solution. ### Non-goals [optional] `None` ## Proposal 1. When volume backup policy is specified: - `if-not-present`: Longhorn will create a backup for volumes that do not have an existing backup. - `always`: Longhorn will create a backup for all volumes, regardless of their existing backups. - `disabled`: Longhorn will not create any backups for volumes. 1. If a volume backup policy is not specified, the policy will be automatically set to `if-not-present`. This ensures that volumes without any existing backups will be backed up during the Longhorn system backup. ### User Stories As a user, I want the ability to specify the volume backup policy when creating the Longhorn system backup. This will allow me to define how volumes should be backed up according to my scenario. - **Scenario 1: if-not-present Policy:** When I set the volume backup policy to `if-not-present`, I expect Longhorn to create a backup for volumes that do not already have a backup. - **Scenario 2: always Policy:** When I set the volume backup policy to `always`, I expect Longhorn to create backups for all volumes, regardless of whether they already have a backup. - **Scenario 3: disabled Policy:** When I set the volume backup policy to `disabled`, I expect Longhorn to not create any backups for the volumes. In cases where I don't explicitly specify the volume backup policy during the system backup configuration, I expect Longhorn to automatically apply the `if-not-present` policy as the default. ### User Experience In Detail To set the volume backup policy, users can set the volume backup policy when creating the system backup through the UI. Alternatively, users can specify it in the manifest when creating the SystemBackup custom resource using the kubectl command. In scenarios where no specific volume backup policy is provided, Longhorn will automatically set the policy as `if-not-present`. ### API changes Add a new `volumeBackupPolicy` field to the HTTP request and response payload. ## Design ### Implementation Overview #### SystemBackup Custom Resource - Introduce a new `volumeBackupPolicy` field. This field allows user to specify the volume backup policy. - Add a new state (phase) called `CreatingVolumeBackups` to track the progress of volume backup creation during the Longhorn system backup. #### CreatingVolumeBackups phase - Iterate through each Longhorn volume. - If the policy is `if-not-present`, create a volume snapshot and backup only for volumes that do not already have a backup (lastBackup is empty). - If the policy is `always`, create a volume snapshot and backup for all volumes, regardless of their existing backups. - If the policy is `disabled`, skip the volume backup creation step for all volumes and proceed to the next phase. - Wait for all volume backups created by the SystemBackup to finish (completed or error state) before proceeding to the next phase (Generating or Error). Backup will have timeout limit of 24 hours. Any of the backups failure will lead the SystemBackup to and Error state. #### Mutate empty volume backup policy When the volume backup policy is not provided in the SystemBackup custom resource, automatically set the policy to `if-not-present`. ### Test plan 1. When the volume backup policy is `if-not-present`, the system backup should only create volume backup when there is no existing backup in Volume. 1. When the volume backup policy is `always`, the system backup should create volume backup regardless of the existing backup. 1. When the volume backup policy is `disabled`, the system backup should not create volume backup. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230601-forcibly-activate-a-restoring-dr-volume.md ================================================ # Forcibly Activate A Restoring/DR Volume ## Summary When users try to activate a restoring/DR volume with some replicas failed for some reasons, the volume will be stuck in attaching state and users can not do anything except deleting the volume. However the volume still can be rebuilt back to be normal as long as there is a healthy replica. To improve user experience, Longhorn should forcibly activate a restoring/DR volume if there is a healthy replica and users allow the creation of a degraded volume. ### Related Issues https://github.com/longhorn/longhorn/issues/1512 ## Motivation ### Goals Allow users to activate a restoring/DR volume as long as there is a healthy replica and the volume works well. ### Non-goals [optional] `None` ## Proposal Forcibly activate a restoring/DR volume if there is a healthy replica and users enable the global setting `allow-volume-creation-with-degraded-availability`. ### User Stories Users can activate a restoring/DR volume by the CLI `kubectl` or Longhorn UI and the volume could work well. ### User Experience In Detail #### Prerequisites Set up two Kubernetes clusters. These will be called cluster A and cluster B. Install Longhorn on both clusters, and set the same backup target on both clusters. 1. In the cluster A, make sure the original volume X has a backup created or has recurring backups scheduled. 2. In backup page of cluster B, choose the backup volume X, then create disaster recovery volume Y. #### Kubectl User set `volume.spec.Standby` to `false` by editing the volume CR or the manifest creating the volume to activate the volume. #### Longhorn UI UI has click `Activate Disaster Recovery Volume` button in `Volume` or `Volume Details` pages to activate the volume. ### API changes `None` ## Design ### Implementation Overview 1. Check if `volume.Spec.Standby` is set to `false` 2. Get the global setting `allow-volume-creation-with-degraded-availability` 3. Activate the DR volume if `allow-volume-creation-with-degraded-availability` is set to `true` and there are one or more ready replicas. ### Test plan #### Test Forcibly Activated A Restoring/DR Volume 1. Create a DR volume 2. Set the global setting `concurrent-replica-rebuild-per-node-limit` to be 0 3. Failed some replicas 4. Check if there is at least one healthy replica 5. Call the API `activate` 6. The volume could be activated 7. Attach the volume to a node and check if data is correct ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230616-automatic-offline-replica-rebuild.md ================================================ # Automatic Offline Replica Rebuilding ## Summary Currently, Longhorn does not have the capability to support online replica rebuilding for volumes utilizing the V2 Data Engine. However, an automatic offline replica rebuilding mechanism has been implemented as a solution to address this limitation. ### Related Issues https://github.com/longhorn/longhorn/issues/6071 ## Motivation ### Goals 1. Support volumes using v2 data engine ### Non-goals 2. Support volumes using v1 data engine ## Proposal ## User Stories In the event of abnormal power outages or network partitions, replicas of a volume may be lost, resulting in volume degradation. Unfortunately, volumes utilizing the v2 data engine do not currently have the capability for online replica rebuilding. As a solution to address this limitation, Longhorn has implemented an automatic offline replica rebuilding mechanism. When a degraded volume is detached, this mechanism places the volume in maintenance mode and initiates the rebuilding process. After the rebuilding is successfully completed, the volume is detached according to the user's specified expectations. ### User Experience In Details - If a volume using the v2 data engine is degraded, the online replica rebuilding process is currently unsupported. - If offline replica rebuilding feature is enabled when one of the conditions is met - Global setting `offline-replica-rebuild` is `enabled` and `Volume.Spec.OfflineReplicaRebuilding` is `ignored` - `Volume.Spec.OfflineReplicaRebuilding` is `enabled` The volume's `Spec.OfflineReplicaRebuildingRequired` is set to `true` if a volume is degraded. - When a degraded volume is detached, this mechanism places the volume in maintenance mode and initiates the rebuilding process. After the rebuilding is successfully completed, the volume is detached according to the user's specified expectations. - If a user attaches the volume without enabling maintenance mode while the replica rebuilding process is in progress, the ongoing replica rebuilding operation will be terminated. ## Design ### Implementation Overview **Settings** - Add global setting `offline-replica-rebuilding`. Default value is `enabled`. The available options are: - `enabled` - `disable` **CRD** - Add `Volume.Spec.OfflineReplicaRebuilding`. The available options are: - ignored`: The volume's offline replica rebuilding behavior follows the settings defined by the global setting `offline-replica-rebuilding`. - `enabled`: Offline replica rebuilding of the volume is always enabled. - `disabled`: Offline replica rebuilding of the volume is always disabled. - Add `Volume.Status.OfflineReplicaRebuildingRequired` **Controller** - Add `volume-rebuilding-controller` for creating and deleting `volume-rebuilding-controller` attachment ticket. **Logics** 1. A volume-controller sets 'Volume.Status.OfflineReplicaRequired' to 'true' when it realizes a v2 data engine is degraded. 2. If a volume's `Volume.Status.OfflineReplicaRebuildingRequired` is `true`, volume-rebuilding-controller creates a `volume-rebuilding-controller` attachment ticket with frontend disabled and lower priority than tickets with workloads. 3. When the volume is detached, volume-attachment-controller attaches the volume with a `volume-rebuilding-controller` attachment ticket in maintenance mode. 4. volume-controller triggers replica rebuilding. 5. After finishing the replica rebuilding, the volume-controller sets `Volume.Status.OfflineReplicaRebuildingRequired` to `false` if a number of healthy replicas is expected. 6. volume-rebuilding-controller deletes the 'volume-rebuilding-controller' attachment ticket. 7. volume-attachment-controller is aware of the deletion of the `volume-rebuilding-controller` attachment ticket, which causes volume detachment. ### Test Plan ### Integration Tests 1. Degraded Volume lifecycle (creation, attachment, detachment and deletion) and automatic replica rebuilding ================================================ FILE: enhancements/20230619-spdk-engine.md ================================================ # SPDK Engine ## Summary Longhorn will take advantage of SPDK to launch the second version engine with higher performance. ### Related Issues https://github.com/longhorn/longhorn/issues/5406 https://github.com/longhorn/longhorn/issues/5282 https://github.com/longhorn/longhorn/issues/5751 ## Motivation ### Goals 1. Have a set of APIs that talks with spdk_tgt to operate SPDK components. 2. Launch a control panel that manage and operate SPDK engines and replica. ## Proposal 1. The SPDK engine architecture is different from the legacy engine: 1. Unlike the legacy engine, the data flow will be taken over by SPDK. The new engine or replica won't directly touch the data handling. The new engine or replica is actually one or a set of SPDK components handled by spdk_tgt. 2. Since the main task is to manage SPDK components and abstract them as Longhorn engines or replicas, we can use a single service rather than separate processes to launch and manage engine or replicas. 3. As SPDK handles the disks by itself, the disk management logic should be moved to SPDK engine service as well. 2. The abstraction of SPDK engine and replica: 1. A data disk will be abstracted as an aio bdev + a lvstore. 2. Each snapshot or volume head file is a logical volume (lvol) inside a lvstore. 3. A remote replica is finally exposed as a NVMe-oF subsystem, in which the corresponding SPDK lvol stand behind. While a local replica is just a lvol. 4. An engine backend is actually a SPDK RAID1 bdev, which may consist of multiple attached replica NVMe-oF subsystems and local lvol. 5. An engine frontend is typically a NVMe-oF initiator plus a NVMe-oF subsystem of the RAID bdev. 3. Do spdk_tgt initializations during instance manager startup. ### User Stories #### Launch SPDK volumes Before the enhancement, users need to launch a RAID1 bdev then expose it as a NVMe-oF initiator as the Longhorn SPDK engine manually by following [the doc](https://github.com/longhorn/longhorn-spdk-engine/wiki/How-to-setup-a-RAID1-block-device-with-SPDK). Besides, rebuilding replicas would be pretty complicated. After the enhancement, users can directly launch and control Longhorn SPDK engine via the gRPC SPDK engine service. And the rebuilding can be triggered and handled automatically. ### API Changes - The new gRPC SPDK engine service: - Replica: | API | Caller | Input | Output | Comments | | --- | --- | --- | --- | --- | | Create | Instance manager proxy | name, lvsName, lvsUUID string, specSize uint64, exposeRequired bool | err error | Create a new replica or start an existing one | | Delete | Instance manager proxy | name string, cleanupRequired bool | err error | Remove or stop an existing replica | | List | Instance manager proxy | | replicas map\[string\]Replica, err error | Get all abstracted replica info from the cache of the SPDK engine service | | Get | Instance manager proxy | | replica Replica, err error | Get the abstracted replica info from the cache of the SPDK engine service | | Watch | Instance manager proxy | | ReplicaStream, err error | Establish a streaming for the replica update notification | | SnapshotCreate | Instance manager proxy | name, snapshotName string | err error | | | SnapshotDelete | Instance manager proxy | name, snapshotName string | err error | | | Rebuilding APIs | The engine inside one gRPC SPDK engine service | | | This set of APIs is responsible for starting and finishing the rebuilding for source replica or destination replica. And it help start data transmission from src to dst | - Engine: | API | Caller | Input | Output | Comments | | --- | --- | --- | --- | --- | | Create | Instance manager proxy | name, lvsName, lvsUUID string, specSize uint64, exposeRequired bool | err error | Start a new engine and connect it with corresponding replicas | | Delete | Instance manager proxy | name string, cleanupRequired bool | err error | Stop an existing engine | | List | Instance manager proxy | | engines map\[string\]Engine, err error | Get the abstracted engine info from the cache of the SPDK engine service | | Get | Instance manager proxy | | engine Engine, err error | Get the abstracted engine info from the cache of the SPDK engine service | | Watch | Instance manager proxy | | EngineStream, err error | Establish a streaming for the engine update notification | | SnapshotCreate | Instance manager proxy | name, snapshotName string | err error | | | SnapshotDelete | Instance manager proxy | name, snapshotName string | err error | | | ReplicaAdd | Instance manager proxy | engineName, replicaName, replicaAddress string | err error | Find a healthy RW replica as source replica then rebuild the destination replica. To rebuild a replica, the engine will call rebuilding start and finish APIs for both replicas and launch data transmission | | ReplicaDelete | Instance manager proxy | engineName, replicaName, replicaAddress string | err error | Remove a replica from the engine | - Disk: | API | Caller | Input | Output | Comments | | --- | --- | --- | --- | --- | | Create | Instance manager proxy | diskName, diskUUID, diskPath string, blockSize int64 | disk Disk, err error | Use the specified block device as blob store | | Delete | Instance manager proxy | diskName, diskUUID string | err error | Remove a store from spdk_tgt | | Get | Instance manager proxy | diskName string | disk Disk, err error | Detect the store status and get the abstracted disk info from spdk_tgt | ## Design ### Implementation Overview #### [Go SPDK Helper](https://github.com/longhorn/go-spdk-helper): - The SPDK Target is exposed as a [JSON-RPC service](https://spdk.io/doc/jsonrpc.html). - Instead of using the existing sample python script [rpc_http_proxy](https://spdk.io/doc/jsonrpc_proxy.html), we will have a helper repo similar to [longhorn/go-iscsi-helper](https://github.com/longhorn/go-iscsi-helper) to talk with spdk_tgt over Unix domain socket `/var/tmp/spdk.sock`.. - The SPDK target config and launching. Then live upgrade, and shutdown if necessary/possible. - The JSON RPC client that directly talks with spdk_tgt. - The exposed Golang SPDK component operating APIs. e.g., lvstore, lvol, RAID creation, deletion, and list. - The NVMe initiator handling Golang APIs (for the engine frontend). #### [SPDK Engine](https://github.com/longhorn/go-spdk-helper): - Launch a gRPC server as the control panel. - Have a goroutine that periodically check and update engine/replica caches. - Implement the engine/replica/disk APIs listed above. - Notify upper layers about the engine/replica update via streaming. #### Instance Manager: - Start spdk_tgt on demand. - Update the proxy service so that it forwards SPDK engine/replica requests to the gRPC service. ### Test Plan #### Integration tests 1. Starting and stopping related tests: If Longhorn can start or stop one engine + multiple replicas correctly. 2. Basic IO tests: If Data can be r/w correctly. And if data still exists after restart. 3. Basic snapshot tests: If snapshots can be created and keeps identical among all replicas. If a snapshot can be deleted from all replicas. If snapshot revert work. #### Manual tests 1. SPDK volume creation/deletion/attachment/detachment tests. 2. Basic IO tests: If Data can be r/w correctly when volume is degraded or healthy. And if data still exists after restart. 3. Basic offline rebuilding tests. ### Upgrade strategy This is an experimental engine. We do not need to consider the upgrade or compatibility issues now. ================================================ FILE: enhancements/20230718-disk-anti-affinity.md ================================================ # Disk Anti-Affinity ## Summary Longhorn supports multiple disks per node, but there is currently no way to ensure that two replicas for the same volume that schedule to the same node end up on different disks. In fact, the replica scheduler currently doesn't make any attempt achieve this goal, even when it is possible to do so. With the addition of a Disk Anti-Affinity feature, the Longhorn replica scheduler will attempt to schedule two replicas for the same volume to different disks when possible. Optionally, the scheduler will refuse to schedule a replica to a disk that has another replica for the same volume. Although the comparison is not perfect, this enhancement can be thought of as enabling RAID 1 for Longhorn (mirroring across multiple disks on the same node). See the [Motivation section](#motivation) for potential benefits. ### Related Issues - https://github.com/longhorn/longhorn/issues/3823 - https://github.com/longhorn/longhorn/issues/5149 ### Existing Related Features #### Replica Node Level Soft Anti-Affinity Disabled by default. When disabled, prevents the scheduling of a replica to a node with an existing healthy replica of the same volume. Can also be set at the volume level to override the global default. #### Replica Zone Level Soft Anti-Affinity Enabled by default. When disabled, prevents the scheduling of a replica to a zone with an existing healthy replica of the same volume. Can also be set at the volume level to override the global default. ## Motivation - Large, multi-node clusters will likely not benefit from this enhancement. - Single-node clusters and small, multi-node clusters (on which the number of replicas per volume exceeds the number of available nodes) will experience: - Increased data durability. If a single disk fails, a healthy replica will still exist on an disk that has not failed. - Increased data availability. If a single disk on a node becomes unavailable, but the node itself remains healthy, at least one replica remains healthy. On a single-node cluster, this can directly prevent a volume crash. On a small, multi-node cluster, this can prevent a future volume crash due to the loss of a different node. ### Goals - In all situations, the Longhorn replica scheduler will make a best effort to ensure two replicas for the same volume do not schedule to the same disk. - Optionally, the scheduler will refuse to schedule a replica to a disk that has another replica of the same volume. ## Proposal ### User Stories #### Story 1 My cluster consists of a single node with multiple attached SSDs. When I create any new volume, I want replicas to distribute across these disks so that I can recover from n - 1 disk failures. If there are not as many available disks as desired replicas, I want Longhorn to do the best it can. #### Story 2 My cluster consists of a single node with multiple attached SSDs. When I create any new volume, I want replicas to distribute across these disks so that I can recover from n - 1 disk failure. If there are not as many available disks as desired replicas, I want scheduling to fail obviously. It is important that I know my volumes aren't being protected so I can take action. #### Story 3 My cluster consists of a single node with multiple attached SSDs. When I create a specific, high-priority volume, I want replicas to distribute across these disks so that I can recover from n - 1 disk failure. If there are not as many available disks as desired replicas, I want scheduling to fail obviously. It is important that I know high-priority volume isn't being protected so I can take action. ### User Experience In Detail ### API changes Introduce a new Replica Disk Level Soft Anti-Affinity setting with the following definition. By default, set it to `true`. While it is generally desirable to schedule replicas to different disks, it would break with existing behavior to refuse to schedule replicas when different disks are not available. ```golang SettingDefinitionReplicaDiskSoftAntiAffinity = SettingDefinition{ DisplayName: "Replica Disk Level Soft Anti-Affinity", Description: "Allow scheduling on disks with existing healthy replicas of the same volume", Category: SettingCategoryScheduling, Type: SettingTypeBool, Required: true, ReadOnly: false, Default: "true", } ``` Introduce a new `spec.replicaDiskSoftAntiAffinity` volume field. By default, set it to `ignored`. Similar to the existing `spec.replicaSoftAntiAffinity` and `spec.replicaSoftZoneAntiAffinityFields`, override the global setting if this field is set to `enabled` or `disabled`. ```yaml replicaDiskSoftAntiAffinity: description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk. enum: - ignored - enabled - disabled type: string ``` ## Design ### Implementation Overview The current replica scheduler does the following: 1. Determines which nodes a replica can be scheduled to based on node condition and the `ReplicaSoftAntiAffinity` and `ReplicaZoneSoftAntiAffinity` settings. 1. Creates a list of all schedulable disks on these nodes. 1. Chooses the disk with the most available space for scheduling. Add a step so that the replica scheduler: 1. Determines which nodes a replica can be scheduled to based on node condition and the `ReplicaSoftAntiAffinity` and `ReplicaZoneSoftAntiAffinity` settings. 1. Creates a list of all schedulable disks on these nodes. 1. Filters the list to include only disks with the least number of existing matching replicas and optionally only disks with no existing matching replicas. 1. Chooses the disk from the filtered list with the most available space for scheduling. ### Test plan Minimally implement two new test cases: 1. In a cluster that includes nodes with multiple available disks, create a volume with `spec.replicaSoftAntiAffinity = true`, `spec.replicaDiskSoftAntiAffinity = true`, and `numberOfReplicas` equal to the total number of disks in the cluster. Confirm that each replica schedules to a different disk. It may be necessary to tweak additional factors. For example, ensure that one disk has enough free space that the old scheduling behavior would assign two replicas to it instead of distributing the replicas evenly among the disks. 1. In a cluster that includes nodes with multiple available disks, create a volume with `spec.replicaSoftAntiAffinity = true`, `spec.replicaDiskSoftAntiAffinity = false`, and `numberOfReplicas` equal to one more than the total number of disks in the cluster. Confirm that a replica fails to schedule. Previously, multiple replicas would have scheduled to the same disk and no error would have occurred. ### Upgrade strategy The Replica Disk Level Soft Anti-Affinity setting defaults to `true` to maintain backwards compatibility. It if is set to `false``, new replicas that require scheduling will follow the new behavior. The `spec.replicaDiskSoftAntiAffinity` volume field defaults to `ignored` to maintain backwards compatibility. If it is set to `enabled` on a volume, new replicas for that volume that require scheduling will follow the new behavior. ================================================ FILE: enhancements/20230807-backingimage-backup-support.md ================================================ # BackingImage Backup Support ## Summary This feature enables Longhorn to backup the BackingImage to backup store and restore it. ### Related Issues - [FEATURE] Restore BackingImage for BackupVolume in a new cluster [#4165](https://github.com/longhorn/longhorn/issues/4165) ## Motivation ### Goals - When a Volume with a BackingImage being backed up, the BackingImage will also be backed up. - User can manually back up the BackingImage. - When restoring a Volume with a BackingImage, the BackingImage will also be restored. - User can manually restore the BackingImage. - All BackingImages are backed up in blocks. - If the block contains the same data, BackingImages will reuse the same block in backup store instead of uploading another identical one. ## Proposal ### User Stories With this feature, there is no need for user to manually handle BackingImage across cluster when backing up and restoring the Volumes with BackingImages. ### User Experience In Detail Before this feature: The BackingImage will not be backed up automatically when backing up a Volume with the BackingImage. So the user needs to prepare the BackingImage again in another cluster before restoring the Volume back. After this feature: A BackingImage will be backed up automatically when a Volume with the BackingImage is being backed up. User can also manually back up a BackingImage independently. Then, when the Volume with the BackingImage is being restored from backup store, Longhorn will restore the BackingImage at the same time automatically. User can also manually restore the BackingImage independently. This improve the user experience and reduce the operation overhead. ## Design ### Implementation Overview #### Backup BackingImage - BackupStore - Backup `BackingImage` is not the same as backup `Volume` which consists of a series of `Snapshots`. Instead, a `BackingImage` already has all the blocks we need to backup. Therefore, we don't need to find the delta between two `BackingImages` like what we do for`Snapshots` which delta might exist in other `Snapshots` between the current `Snapshot` and the last backup `Snapshot`. - All the `BackingImages` share the same block pools in backup store, so we can reuse the blocks to increase the backup speed and save the space. This can happen when user create v1 `BackingImage`, use the image to add more data and then export another v2 `BackingImage`. - For restoration, we still restore fully on one of the ready disk. - Different from `Volume` backup, `BackingImage` does not have any size limit. It can be less than 2MB or not a multiple of 2MB. Thus, the last block might not be 2MB. - When backing up `BackingImage` 1. `preload()`: the BackingImage to get the all the sectors that have data inside. 2. `createBackupBackingMapping()`: to get all the blocks we need to backup - Block: offset + size (2MB for each block, last block might less than 2MB) 3. `backupMappings()`: write the block to the backup store - if the block is already in the backup store, skip it. 4. `saveBackupBacking()`: save the metadata of the `BackupBackingImage` including the block mapping to the backup store. Mapping needs to include block size. - When restoring `BackingImage` - `loadBackupBacking()`: load the metadata of the `BackupBackingImage` from the backup store - `populateBlocksForFullRestore() + restoreBlocks()`: based on the mapping, write the block data to the correct offset. - We backup the blocks in async way to increase the backup speed. - For qcow2 `BackingImage`, the format is not the same as raw file, we can't detect the hole and the data sector. So we back up all the blocks. #### Backup BackingImage - Controller 1. Add a new CRD `backupbackingimage.longhorn.io` ```go type BackupBackingImageSpec struct { SyncRequestedAt metav1.Time `json:"syncRequestedAt"` UserCreated bool `json:"userCreated"` Labels map[string]string `json:"labels"` } type BackupBackingImageStatus struct { OwnerID string `json:"ownerID"` Checksum string `json:"checksum"` URL string `json:"url"` Size string `json:"size"` Labels map[string]string `json:"labels"` State BackupBackingImageState `json:"state"` Progress int `json:"progress"` Error string `json:"error,omitempty"` Messages map[string]string `json:"messages"` ManagerAddress string `json:"managerAddress"` BackupCreatedAt string `json:"backupCreatedAt"` LastSyncedAt metav1.Time `json:"lastSyncedAt"` CompressionMethod BackupCompressionMethod `json:"compressionMethod"` } ``` ```go type BackupBackingImageState string const ( BackupBackingImageStateNew = BackupBackingImageState("") BackupBackingImageStatePending = BackupBackingImageState("Pending") BackupBackingImageStateInProgress = BackupBackingImageState("InProgress") BackupBackingImageStateCompleted = BackupBackingImageState("Completed") BackupBackingImageStateError = BackupBackingImageState("Error") BackupBackingImageStateUnknown = BackupBackingImageState("Unknown") ) ``` - Field `Spec.UserCreated` indicates whether this Backup is created by user to create the backup in backupstore or it is synced from backupstrore. - Field `Status.ManagerAddress` indicates the address of the backing-image-manager running BackingImage backup. - Field `Status.Checksum` records the checksum of the BackingImage. Users may create a new BackingImage with the same name but different content after deleting an old one or there is another BackingImage with the same name in another cluster. To avoid the confliction, we use checksum to check if they are the same. - If cluster already has the `BackingImage` with the same name as in the backup store, we still create the `BackupBackingImage` CR. User can use the checksum to check if they are the same. Therefore we don't use `UUID` across cluster since user might already prepare the same BackingImage with the same name and content in another cluster. 2. Add a new controller `BackupBackingImageController`. - Workflow - Check and update the ownership. - Do cleanup if the deletion timestamp is set. - Cleanup the backup `BackingImage` on backup store - Stop the monitoring - If `Status.LastSyncedAt.IsZero() && Spec.BackingImageName != ""` means **it is created by the User/API layer**, we need to do the backup - Start the monitor - Pick one `BackingImageManager` - Request `BackingImageManager` to backup the `BackingImage` by calling `CreateBackup()` grpc - Else it means the `BackupBackingImage` CR is created by `BackupTargetController` and the backup `BackingImage` already exists in the remote backup target before the CR creation. - Use `backupTargetClient` to get the info of the backup `BackingImage` - Sync the status 3. In `BackingImageManager - manager(backing_image.go)` - Implement `CreateBackup()` grpc - Backup `BackingImage` to backup store in blocks 4. In controller `BackupTargetController` - Workflow - Implement `syncBackupBackingImage()` function - Create the `BackupBackingImage` CRs whose name are in the backup store but not in the cluster - Delete the `BackupBackingImage` CRs whose name are in the cluster but not in the backup store - Request `BackupBackingImageController` to reconcile those `BackupBackingImage` CRs 5. Add a backup API for `BackingImage` - Add new action `backup` to `BackingImage` (`"/v1/backingimages/{name}"`) - create `BackupBackingImage` CR to init the backup process - if `BackupBackingImage` already exists, it means there is already a `BackupBackingImage` in backup store, user can check the checksum to verify if they are the same. - API Watch: establish a streaming connection to report BackupBackingImage info. 6. Trigger - Back up through `BackingImage` operation manually - Back up `BackingImage` when user back up the volume - in `SnapshotBackup()` API - we get the `BackingImage` of the `Volume` - back up `BackingImage` if the `BackupBackingImage` does not exist #### Restoring BackingImage - Controller 2. Add new data source type `restore` for `BackingImageDataSource` ```go type BackingImageDataSourceType string const ( BackingImageDataSourceTypeDownload = BackingImageDataSourceType("download") BackingImageDataSourceTypeUpload = BackingImageDataSourceType("upload") BackingImageDataSourceTypeExportFromVolume = BackingImageDataSourceType("export-from-volume") BackingImageDataSourceTypeRestore = BackingImageDataSourceType("restore") DataSourceTypeRestoreParameterBackupURL = "backup-url" ) // BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source type BackingImageDataSourceSpec struct { NodeID string `json:"nodeID"` UUID string `json:"uuid"` DiskUUID string `json:"diskUUID"` DiskPath string `json:"diskPath"` Checksum string `json:"checksum"` SourceType BackingImageDataSourceType `json:"sourceType"` Parameters map[string]string `json:"parameters"` FileTransferred bool `json:"fileTransferred"` } ``` 3. Create BackingImage APIs - No need to change - Create BackingImage CR with `type=restore` and `restore-url=${URL}` - If BackingImage already exists in the cluster, user can use checksum to verify if they are the same. 4. In `BackingImageController` - No need to change, it will create the `BackingImageDataSource` CR 5. In `BackingImageDataSourceController` - No need to change, it will create the `BackingImageDataSourcePod` to do the restore. 6. In `BackingImageManager - data_source` - When init the service, if the type is `restore`, then restore from `backup-url` by requesting sync service in the same pod. ```go requestURL := fmt.Sprintf("http://%s/v1/files", client.Remote) req, err := http.NewRequest("POST", requestURL, nil) q := req.URL.Query() q.Add("action", "restoreFromBackupURL") q.Add("url", backupURL) q.Add("file-path", filePath) q.Add("uuid", uuid) q.Add("disk-uuid", diskUUID) q.Add("expected-checksum", expectedChecksum) ```` - In `sync/service` implement `restoreFromBackupURL()` to restore the `BackingImage` from backup store to the local disk. 7. In `BackingImageDataSourceController` - No need to change, it will take over control when `BackingImageDataSource` status is `ReadyForTransfer`. - If it failed to restore the `BackingImage`, the status of the `BackingImage` will be failed and `BackingImageDataSourcePod` will be cleaned up and retry with backoff limit like `type=download`. The process is the same as other `BackingImage` creation process. 8. Trigger - Restore through `BackingImage` operation manually - Restore when user restore the `Volume` with `BackingImage` - Restoring a Volume is actually requesting `Create` a Volume with `fromBackup` in the spec - In `Create()` API we check if the `Volume` has `fromBackup` parameters and has `BackingImage` - Check if `BackingImage` exists - Check and restore `BackupBackingImage` if `BackingImage` does not exist - Restore `BackupBackingImage` by creating `BackingImage` with type `restore` and `backupURL` - Then Create the `Volume` CR so the admission webhook won't failed because of missing `BackingImage` ([ref](https://github.com/longhorn/longhorn-manager/blob/master/webhook/resources/volume/validator.go#L86)) - Restore when user create `Volume` through `CSI` - In `CreateVolume()` we check if the `Volume` has `fromBackup` parameters and has `BackingImage` - In `checkAndPrepareBackingImage()`, we restore `BackupBackingImage` by creating `BackingImage` with type `restore` and `backupURL` #### API and UI changes In Summary 1. `longhorn-ui`: - Add a new page of `BackupBackingImage` like `Backup` - The columns on `BackupBackingImage` list page should be: `Name`, `Size`, `State`, `Created At`, `Operation`. - `Name` can be clicked and will show `Checksum` of the `BackupBackingImage` - `State`: `BackupBackingImageState` of the `BackupBackingImage` CR - `Operation` includes - `restore` - `delete` - Add a new operation `backup` for every `BackingImage` in the `BackingImage` page 2. `API`: - Add new action `backup` to `BackingImage` (`"/v1/backingimages/{name}"`) - create `BackupBackingImage` CR to init the backup process - `BackupBackingImage` - `GET "/v1/backupbackingimages"`: get all `BackupBackingImage` - API Watch: establish a streaming connection to report `BackupBackingImage` info change. ### Test plan Integration tests 1. `BackupBackingImage` Basic Operation - Setup - Create a `BackingImage` ``` apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot namespace: longhorn-system spec: sourceType: download sourceParameters: url: https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.raw checksum: 304f3ed30ca6878e9056ee6f1b02b328239f0d0c2c1272840998212f9734b196371560b3b939037e4f4c2884ce457c2cbc9f0621f4f5d1ca983983c8cdf8cd9a ``` - Setup the backup target - Back up `BackingImage` by applying the yaml - yaml ```yaml apiVersion: longhorn.io/v1beta2 kind: BackupBackingImage metadata: name: parrot namespace: longhorn-system spec: userCreated: true labels: usecase: test type: raw ``` - `BackupBackingImage` CR should be complete - You can get the backup URL from `Status.URL` - Delete the `BackingImage` in the cluster - Restore the `BackupBackingImage` by applying the yaml ```yaml apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot-restore namespace: longhorn-system spec: sourceType: restore sourceParameters: # change to your backup URL # backup-url: nfs://longhorn-test-nfs-svc.default:/opt/backupstore?backingImage=parrot backup-url: s3://backupbucket@us-east-1/?backingImage=parrot concurrent-limit: "2" checksum: 304f3ed30ca6878e9056ee6f1b02b328239f0d0c2c1272840998212f9734b196371560b3b939037e4f4c2884ce457c2cbc9f0621f4f5d1ca983983c8cdf8cd9a ``` - Checksum should be the same 2. Back up `BackingImage` when backing up and restoring Volume - Setup - Create a `BackingImage` - Setup the backup target - Create a Volume with the `BackingImage` - Back up the `Volume` - `BackupBackingImage` CR should be created and complete - Delete the `BackingImage` - Restore the Volume with same `BackingImage` - `BackingImage` should be restored and the `Volume` should also be restored successfully - `Volume` checksum is the same Manual tests 1. `BackupBackingImage` reuse blocks - Setup - Create a `BackingImage` A - Setup the backup target - Create a `Volume` with `BackingImage` A, write some data and export to another `BackingImage` B - Back up `BackingImage` A - Back up `BackingImage` B - Check it reuses the blocks when backing up `BackingImage` B (by trace log) ================================================ FILE: enhancements/20230809-support-backup-and-restore-for-volumes-with-v2-data-engine.md ================================================ # Support Backup and Restore For Volumes with V2 Data Engine ## Summary Support backup and full restore for volumes with v2 data engine. ### Related Issues - [[FEATURE] SPDK volumes support volume backup/restore](https://github.com/longhorn/longhorn/issues/6138) ## Motivation ### Goals - Leverage the existing backup and restore frameworks - Back up volumes with v2 data engine - Fully restore volumes with v2 data engine - Support fully restoring v2 volume backup to v1/v2 volume ### Non-goals - Incremental restore volumes with v2 data engine ## Proposal ## User Stories Backup and restore are essential for data recovery. Longhorn system can back up volumes with v1 data engine. However, it does not support snapshot backup for v2 data engine yet. For better user experience, backup and restore operations should be consistent across both data engines. Additionally, a v2 volume can support restoring to an either v1 or v2 volume. ### User Experience In Details Backup and restore operations should be consistent across both data engines. ### API Changes - engine-proxy - Add `BackupRestoreFinish` for wrapping up the volume restore - spdk - Add `bdev_lvol_get_fragmap` API - Get a fragmap for a specific segment of a logical volume (lvol) using the provided offset and size. A fragmap is a bitmap that records the allocation status of clusters. A value of "1" indicates that a cluster is allocated, whereas "0" signifies that a cluster is unallocated. - Add `creation_time` xattr in lvols - The `creation_time` is represented as a snapshot creation time. - Add `bdev_lvol_get_xattr` and `bdev_lvol_set_xattr` - spdk-go-helper - Add `BdevLvolGetFragmap` for `bdev_lvol_get_fragmap` - Add `BdevGetXattr` for `bdev_lvol_get_xattr` - Add `BdevSetXattr` for `bdev_lvol_set_xattr` ### UI Changes - In `UI > Backup > Volume Operations > Restore` page, add `Data Engine` option. ## Design Longhorn can incrementally back up volumes that use the v1 data engine. Incremental backups are based on fiemap, which are file extent mappings of the disk files. This means that only the changes to the volume since the last backup are backed up, which can save time and space. However, snapshots of volumes that use the v2 data engine are SPDK lvols, which do not support fiemap. Instead, we can use the concept of fiemap to leverage the fragmentation map (fragmap) instead. A fragmap is a mapping of data and hole segments in a volume. This means that we can use the fragmap to identify the changed segments of a v2 volume and back them up, even though fiemap is not supported. Here are some additional details about fiemap and fragmaps: - fiemap is a Linux kernel API that allows applications to query the file system for the extents of a file. This can be used to efficiently back up files, as only the changed extents need to be backed up. - fragmap is a data structure that stores the fragmentation information for a volume. This information includes the size and location of each data segment in the volume, as well as the size and location of any holes in the volume. ### Implementation Overview - **spdk_tgt** - Add `bdev_lvol_get_fragmap` JSON-RPC method: retrieve the fragmap for a specific portion of the specified lvol bdev - Parameters - lvs_name: The name of the lvstore name - lvol_name: The name of the lvol. - offset: offset the a specific portion of the lvol. Unit: bytes. - size: size the a specific portion of the lvol. Unit: bytes. - The fragmap is internally constructed by incorporating with `spdk_bdev_seek_hole` and `spdk_bdev_seek_data` - The size of each data segment or hole segment is the same as the cluster size of a lvstore - An example of a fragmap of a lvol ![fragmap](image/fragmap.png) - **Backup Flow** The backup flow is explained by the an example. A snapshot chain of a volume is ``` volume-head -> snap003 -> snap002 -> snap001 (backed up) ``` `snap001` is already backed up, and we are going to back up `snap003`. The steps are 1. longhorn-manager's backup controller is aware of the backup resource for the snapshot `snapshot3` and then issues a backup request to the instance-manager by the proxy function `SnapshotBackup()` with the specified volume and snapshot names. 2. After the instance-manager proxy service receives the request, the proxy function transfers the request to the SPDK service by `EngineBackupCreate()`. 3. The SPDK service randomly picks one of the replicas and executes the backup by `ReplicaBackupCreate()`. 4. Expose snapshot (`snap003`) to be a block device locally via NVMe over TCP. The data in the block device is the accumulated data of `snap003` and its descendants. 5. Get the fragmaps of the snapshot to be backed up and its descendants prior to the last backup. In the example, we need to get the fragmaps of `snap003` and `snap002`. 6. Retrieve the fragmap of each lvol using `bdev_lvol_get_fragmap` JSON-RPC method. 7. Overlay all the fragmaps, iterate each bit, and if it's set to 1, read data from the corresponding cluster on the block device and perform a backup. - **Full Restore Flow** 1. The engine controller within Longhorn Manager is informed of a volume restore request. Subsequently, it initiates a restore operation to the instance manager using the BackupRestore() proxy function. This operation includes essential details such as the volume name, backup name, credentials, and more. 2. Once the instance manager's proxy service receives the request, the `EngineRestoreCreate()` proxy function is employed to transfer the request to the SPDK service. 3. Within `EngineRestoreCreate()`, the raid bdev is removed, followed by issuing a replica restore command to the associated replicas through the `ReplicaRestoreCreate()` function. 4. In the `ReplicaRestoreCreate()`, the lvol of the replica is locally exposed as a block device using NVMe over TCP. 5. The subsequent steps involve downloading data blocks and writing them to the block device according to the block mappings. 6. Upon the successful completion of the restoration process, Longhorn Manager sends a request to finalize the restoration to the instance manager via the `EngineRestoreFinish()` function. This function facilitates the recreation of the raid bdev. ## Test Plan - Concurrent volume backup and full restore ## Note[optional] ================================================ FILE: enhancements/20230814-talos-linux-support.md ================================================ # Talos Linux Support ## Summary [Talos Linux is not based on X distro](https://www.talos.dev/v1.4/learn-more/philosophy/#not-based-on-x-distro) >Talos Linux isn’t based on any other distribution. We think of ourselves as being the second-generation of container-optimised operating systems, where things like CoreOS, Flatcar, and Rancher represent the first generation (but the technology is not derived from any of those.) > >Talos Linux is actually a ground-up rewrite of the userspace, from PID 1. We run the Linux kernel, but everything downstream of that is our own custom code, written in Go, rigorously-tested, and published as an immutable, integrated image. The Linux kernel launches what we call machined, for instance, not systemd. There is no systemd on our system. There are no GNU utilities, no shell, no SSH, no packages, nothing you could associate with any other distribution. Currently, Longhorn (at version v1.5.x) does not support Talos Linux as one of the [operating systems (OS)](https://longhorn.io/docs/1.5.0/best-practices/#operating-system) due to reliance on host binaries like BASH and iscsiadm. The goal of this proposal is to enable Longhorn to be installed and functional on Talos Linux clusters. ### Related Issues https://github.com/longhorn/longhorn/issues/3161 ## Motivation We have been approached by and observed a number of Talos Linux users who wish to utilize Longhorn. However, the lack of support for Talos Linux prevents them from using Longhorn as their storage solution. ### Goals The primary goal of this proposal is to introduce support for Talos Linux, allowing users to installed and operate Longhorn on Talos Linux clusters. ### Non-goals [optional] `None` ## Proposal Whenever possible, replace host binary dependencies with alternative solutions: - Develop a common thread namespace switch package that can be utilized by projects requiring interaction with the host. This is necessary due to the absence of GNU utilities in Talos Linux. For example, Longhorn is unable to utilize nsenter and execute binaries in [lockCmd](https://github.com/longhorn/nsfilelock/blob/2315476ea52e13b4112a1cdb930626f8aa848d09/nsfilelock.go#L67-L68). - Modify the invocation of the `iscsiadm` binary to execute within the `iscsid` process namespace, - In cases where replacing binary invocations is not feasible, leverage the Talos Linux `kubelet` namespace specifically for Talos Linux. For other operating systems, maintain the existing approach. ### User Stories As a user running Talos Linux, I want to be able to use Longhorn as my storage solution. Currently, Longhorn does not support Talos Linux, so I am unable to utilize its features. With this enhancement, I will be able to install and use Longhorn on my Talos Linux cluster. ### User Experience In Detail 1. Provision a Talos Linux cluster. 1. Apply the required machine configurations to the cluster nodes: - Install the iscsi-tool system extension. - Install the util-linux system extension. - Add the /var/lib/longhorn extra mount. 1. Install Longhorn on the Talos Linux cluster. 1. Once Longhorn is successfully installed, access and utilize Longhorn just like users on other supported operating systems. ### API changes `None` ## Design ### Thread Namespace Switch - Create a package in the [longhorn/go-common-lib](https://github.com/longhorn/go-common-libs) repository that can be imported by other projects. - The targeting function execute within a goroutine and utilize the `unix.Setns` to switch to different namespaces. - As Golang being a multi-threaded language, to ensure exclusive use of the thread, lock the thread when switching to a different namespace. - Once the targeting function completes its execution, the thread will switch back to the original namespace and unlock the thread to allow other goroutines to execute. > The primary goal of this implementation is to replace the usage of GNU utilities with Go-based implementations where applicable, particularly in areas such as file handling. ### Binary Dependencies #### Isciadm Longhorn currently assumes that `iscsid` is running on the host. However, in Talos Linux, `iscsid` runs as a Talos extension service in a different namespace. To address this, modify the `iscsiadm` binary invocation to execute within the `iscsid` namespace using `nsenter`. #### fstrim ~Instead of rewriting the `fstrim` binary dependency, leverage the existing `fstrim` binary within the `kubelet` namespace. However, considering that the `kubelet` namespace might not be present in other operating systems, Longhorn needs to switch to the host namespace to retrieve the OS distribution and determine if it is a Talos Linux. If the host is identified as Talos Linux, Longhorn can then switch to the `kubelet` namespace to utilize the existing `fstrim` binary.~ Keep the binary execution in the host namespace, following a discussion with @frezbo from @siderolabs. The Talos team has proposed an approach to include the fstrim in the host as part of `util-linux` extension. #### cryptsetup Keep the current implementation of the binary execution in the host namespace, as the `cryptsetup` binary comes [pre-installed](https://github.com/siderolabs/pkgs/blob/release-1.4/Makefile#L40) in Talos Linux, for the [Disk Encryption](https://www.talos.dev/v1.4/talos-guides/configuration/disk-encryption/). ### Project Adpaptions - Replace the `nsenter` path with the `iscsid` proc path when invoking `iscsiadm` binary. - Replace the `nsenter` path with the `kubelet` proc path when invoking other dependency binaries. This change will be applied only if the operating system distribution is identified as Talos Linux, as other distributions may not have a running `kubelet` process. - For binary invocations are not part of the [binary dependencies](#binary-dependencies), replace them with appropriate Golang libraries. - Utilize the [thread namespace switch implementation](#thread-namespace-switch) to handle the necessary namespace switching when interacting with the host. ### Test plan 1. Update existing test cases that rely on `nsenter` to either correct the namespace or replace them with appropriate Python libraries. 1. Perform regression testing by running existing test cases to ensure that no regressions are introduced. 1. Introduce a new pipeline for testing Longhorn on Talos Linux clusters in https://ci.longhorn.io/. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230815-engine-upgrade-enforcement.md ================================================ # Engine Upgrade Enforcement ## Summary The current Longhorn upgrade process lacks enforcement of the engine version, potentially leading to compatibility issues. To address this concern, we propose the implementation of an Engine Upgrade Enforcement feature. ### Related Issues https://github.com/longhorn/longhorn/issues/5842 ## Motivation Longhorn needs to be able to upgrade safely without risking compatibility issue with older version engine images. Without this enhancement, we are facing various challenges like dealing with potential operation failures and increased maintenance overhead. ### Goals The primary goal of this proposal is to enhance Longhorn's upgrade mechanism by introducing logic that prevents upgrading to Longhorn versions while there are incompatible engine images in use. ### Non-goals [optional] `None` ## Proposal This proposal focuses on preventing users from upgrading to unsupported or incompatible engine versions. This enhancement will build upon the existing pre-upgrade checks to include validation of engine version compatibility. ### User Stories #### Story 1: Preventing Incompatible Upgrades Previously, users had the freedom to continue using an older engine version after a Longhorn upgrade. With the proposed enhancement, the Longhorn upgrade process will be blocked if it includes an incompatible engine version. This will enforce users to manually upgrade the engine to a compatible version before proceeding with the Longhorn upgrade. ### User Experience In Detail User will perform upgrade a usual. Longhorn will examine the compatibility of the current engine version. If the current engine version is incompatible with the target engine version for the upgrade, Longhorn will halt the upgrade process and prompt the user to address the engine version mismatch before proceeding. ### API changes `None` ## Design ### Implementation Overview The implementation approach for this feature will be similar to the [Upgrade Path Enforcement feature](https://github.com/longhorn/longhorn/blob/master/enhancements/20230315-upgrade-path-enforcement.md). Key implementation steps include: 1. Enhance the function [CheckUpgradePathSupported(...)](https://github.com/longhorn/longhorn-manager/blob/v1.5.1/upgrade/util/util.go#L168) to include the new checks. ``` func CheckUpgradePathSupported(namespace string, lhClient lhclientset.Interface) error { if err := CheckLHUpgradePathSupported(namespace, lhClient); err != nil { return err } return CheckEngineUpgradePathSupported(namespace, lhClient, emeta.GetVersion()) } ``` 1. Retrieve the current engine images being used and record the versions. 1. Prevent upgrades if the targeting engine version is detact to be downgrading. 1. Prevent upgrades if the engine image version is lower than [the minimum required version for the new engine image controller API](https://github.com/longhorn/longhorn-engine/blob/v1.5.1/pkg/meta/version.go#L10). ### Test plan - Create unit test for the new logic. - Run manual test to verify the handling of incompatible engine image versions (e.g., Longhorn v1.4.x -> v1.5.x -> v1.6.x.) ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20230821-customize-maximum-recurring-job-retain-number.md ================================================ # Customize Maximum Recurring Job Retain Number ## Summary Hardcoding the `MaxRecurringJobRetain` to be 100 may be inadequate in some cases such as that users try to keep 125 backups on the remote backup target. This feature will make the `MaxRecurringJobRetain` to be customizable by users. **NOTE**: 1. Having more snapshots will consume disk storage capacity. 2. The maximum number of snapshots is 250. It will be failed if recurring job start to create a new snapshot after the number of snapshots is up to 250. 3. 1 and 2 might cause volume rebuilding/taking a snapshot/taking a backup unstable. ### Related Issues - [Longhorn S3 backup retain limit](https://github.com/longhorn/longhorn/discussions/5710) - [Customize MaxRecurringJobRetain](https://github.com/longhorn/longhorn/issues/5713) - [Longhorn snapshot space management](https://github.com/longhorn/longhorn/issues/6563) ## Motivation ### Goals - Add a global setting and users will be able to customize the retain number of a recurring job. ### Non-goals [optional] - Retain number can exceed 250. ## Proposal ### User Stories Users can customize the maximum retain number when creating a recurring job to keep more snapshots or backups or restrict resources consumption especially disk storage capacity. ### User Experience In Detail Users can customize maximum retain number of the recurring job and maximum retain number cannot be exceeded when creating a recurring job. In [Longhorn snapshot space management](https://github.com/longhorn/longhorn/issues/6563) will help users handle the snapshot space management but user sill need to consider the total disk storage capacity as well as the size and number of volumes to determine the retain number. #### Kubectl User can set the global setting `recurring-job-max-retention` value to be a customized number under 250. ```txt kubectl -n longhorn-system edit setting recurring-job-max-retention -- apiVersion: longhorn.io/v1beta2 kind: Setting metadata: annotations: longhorn.io/configmap-resource-version: "6789" creationTimestamp: "2023-08-21T00:00:00Z" generation: 1 name: recurring-job-max-retention namespace: longhorn-system resourceVersion: "87651" uid: ab1f01cf-xxxx-45ab-9999-8ca1ebf61xxx value: "200" ``` #### Helm User can set `recurringJobMaxRetain` value to be a customized number under 250 such as: ```bash helm upgrade longhorn longhorn/longhorn --set defaultSettings.recurringJobMaxRetain=200 ``` #### Longhorn UI There is a global setting named `Recurring Job Maximum Retain Number` on the `Setting`/`General` page of UI. Users can fill out the retain number under 250 and save the setting. ### API changes `None` ## Design ### Implementation Overview 1. Add the global setting `recurring-job-max-retention` and default value is `100`. 2. The number of global setting `recurring-job-max-retention` cannot be exceeded `250` because of the maximum number of snapshots for a volume. ### Test plan 1. Set the global setting `recurring-job-max-retention` to be 251 and it will fail. 2. Set the global setting `recurring-job-max-retention` to be 50 and it will success. 3. Create a recurring job with retain number 50 and it will success. 4. Create a recurring job with retain number 51 and it will fail. ### Upgrade strategy `None` ================================================ FILE: enhancements/20230905-automatically-evict-replicas-while-draining.md ================================================ # Automatically Evict Replicas While Draining ## Summary Currently, Longhorn allows the choice between a number of behaviors (node drain policies) when a node is cordoned or drained: - `Block If Contains Last Replica` ensures the `instance-manager` pod cannot be drained from a node as long as it is the last node with a healthy replica for some volume. Benefits: - Protects data by preventing the drain operation from completing until there is a healthy replica available for each volume available on another node. Drawbacks: - If there is only one replica for the volume, or if its other replicas are unhealthy, the user may need to manually (through the UI) request the eviction of replicas from the disk or node. - Volumes may be degraded after the drain is complete. If the node is rebooted, redundancy is reduced until it is running again. If the node is removed, redundancy is reduced until another replica rebuilds. - `Allow If Last Replica Is Stopped` is similar to the above, but only prevents an `instance-manager` pod from draining if it has the last RUNNING replica. Benefits: - Allows the drain operation to proceed in situations where the node being drained is expected to come back online (data will not be lost) and the replicas stored on the node's disks are not actively being used. Drawbacks: - Similar drawbacks to `Block If Contains Last Replica`. - If, for some reason, the node never comes back, data is lost. - `Always Allow` never prevents an `instance-manager` pod from draining. Benefits: - The drain operation completes quickly without Longhorn getting in the way. Drawbacks: - There is no opportunity for Longhorn to protect data. This proposal seeks to add a fourth and fifth behavior (node drain policy) with the following properties: - `Block For Eviction` ensures the `instance-manager` pod cannot be drained from a node as long as it contains any replicas for any volumes. Replicas are automatically evicted from the node as soon as it is cordoned. Benefits: - Protects data by preventing the drain operation from completing until all replicas have been relocated. - Automatically evicts replicas, so the user does not need to do it manually (through the UI). - Maintains replica redundancy at all times. Drawbacks: - The drain operation is significantly slower than for other behaviors. Every replica must be rebuilt on another node before it can complete. - The drain operation is data-intensive, especially when replica auto balance is enabled, as evicted replicas may be moved back to the drained node when/if it comes back online. - Like all of these policies, it triggers on cordon, not on drain (it is not possible for Longhorn to distinguish between a node that is actively being drained and one that is cordoned for some other reason). If a user regularly cordons nodes without draining them, replicas will be rebuilt pointlessly. - `Block For Eviction If Contains Last Replica` ensures the `instance-manager` pod cannot be drained from a node as long as it is the last node with a healthy replica for some volume. Replicas that meet this condition are automatically evicted from the node as soon as it is cordoned. Benefits: - Protects data by preventing the drain operation from completing until there is a healthy replica available for each volume available on another node. - Automatically evicts replicas, so the user does not need to do it manually (through the UI). - The drain operation is only as slow and data-intensive as is necessary to protect data. Drawbacks: - Volumes may be degraded after the drain is complete. If the node is rebooted, redundancy is reduced until it is running again. If the node is removed, redundancy is reduced until another replica rebuilds. - Like all of these policies, it triggers on cordon, not on drain (it is not possible for Longhorn to distinguish between a node that is actively being drained and one that is cordoned for some other reason). If a user regularly cordons nodes without draining them, replicas will be rebuilt pointlessly. Given the drawbacks, `Block For Eviction` should likely not be the default node drain policy moving forward. However, some users may find it helpful to switch to `Block For Eviction`, especially during cluster upgrade operations. See [user stories](#user-stories) for additional insight. `Block For Eviction If Contains Last Replica` is a more efficient behavior that may make sense as the long-term setting in some clusters. It has largely the same benefits and drawbacks as `Block If Contains Last Replica`, except that it doesn't require users to perform any manual drains. Still, when data is properly backed up and the user is planning to bring a node back online after maintenance, it is costlier than other options. ### Related Issues https://github.com/longhorn/longhorn/issues/2238 ## Motivation ### Goals - Add a new `Block For Eviction` node drain policy as described in the summary. - Ensure that replicas automatically evict from a cordoned node when `Block For Eviction` is set. - Ensure a drain operation can not complete until all replicas are evicted when `Block For Eviction` is set - Document recommendations for when to use `Block For Eviction`. ### Non-goals - Only trigger automatic eviction when a node is actively being drained. It is not possible to distinguish between a node that is only cordoned and one that is actively being drained. ## Proposal ### User Stories #### Story 1 I use Rancher to manage RKE2 and K3s Kubernetes clusters. When I upgrade these clusters, the system upgrade controller attempts to drain each node before rebooting it. If a node contains the last healthy replica for a volume, the drain never completes. I know I can manually evict replicas from a node to allow it to continue, but this eliminates the benefit of the automation. After this enhancement, I can choose to set the node drain policy to `Block For Eviction` before kicking off a cluster upgrade. The upgrade may take a long time, but it eventually completes with no additional intervention. #### Story 2 I am not comfortable with the reduced redundancy `Block If Contains Last Replica` provides while my drained node is being rebooted. Or, I commonly drain nodes to remove them from the cluster and I am not comfortable with the reduced redundancy `Block If Contains Last Replica` provides while a new replica is rebuilt. It would be nice if I could drain nodes without this discomfort. After this enhancement, I can choose to set the node drain policy to `Block For Eviction` before draining a node or nodes. It may take a long time, but I know my data is safe when the drain completes. ### User Experience In Detail ### API changes Add `block-for-eviction` and `block-for-eviction-if-last-replica` options to the `node-drain-policy` setting. The user chooses these options to opt in to the new behavior. Add a `status.autoEvicting` to the `node.longhorn.io/v1beta2` custom resource. This is not a field users can/should interact with, but they can view it via kubectl. NOTE: We originally experimented with a new `status.conditions` entry in the `node.longhorn.io/v1beta2` custom resource with the type `Evicting`. However, this was a bit less natural, because: - Longhorn node conditions generally describe the state a node is in, not what the node is doing. - During normal operation, `Evicting` should be `False`. The Longhorn UI displays a condition in this state with a red symbol, indicating an error state that should be investigated. Deprecate the `replica.status.evictionRequested` field in favor of a new `replica.spec.evictionRequested` field so that replica eviction can be requested by the node controller instead of the replica controller. ## Design ### Implementation Overview The existing eviction logic is well-tested, so there is no reason to significantly refactor it. It works as follows: - The user can set `spec.evictionRequested = true` on a node or disk. - When the replica controller sees `spec.evictionRequested == true` on the node or disk hosting a replica, it sets `status.evictionRequested = true` on that replica. - The volume controller uses `replica.status.evictionRequested == true` to influence replica scheduling/deletion behavior (e.g. rebuild an extra replica to replace the evicting one or delete the evicting one once rebuilding is complete). - The user can set `spec.evictionRequested = false` on a node or disk. - When the replica controller sees `spec.evictionRequested == false` on the node or disk hosting a replica, it sets `replica.status.evictionRequested = false` on that replica. - The volume controller uses `replica.status.evictionRequested == false` to influence replica scheduling/deletion behavior (e.g. don't start a rebuild for a previously evicting replica if one hasn't been started already). - NOTE: If a new replica already started rebuilding as part of an eviction, it continues to rebuild and remains in the cluster even after eviction is canceled. It can be cleaned up manually if desired. Make changes so that: - The node controller (not the replica controller) sets `replica.spec.evictionRequested = true` when: - `spec.evictionRequested == true` on the replica's node (similar to existing behavior moved from the replica controller), OR - `spec.evictionRequested == true` on the replica's disk. (similar to existing behavior moved from the replica controller), OR - `status.Unschedulable == true` on the associated Kubernetes node object and the node drain policy is `block-for-eviction`, OR - `status.Unschedulable == true` on the associated Kubernetes node object, the node drain policy is `block-for-eviction-if-contains-last-replica`, and there are no other PDB-protected replicas for a volume. - Much of the logic currently used by the instance manager controller to recognize PDB-protected replicas is moved to utility functions so both the node and instance manager controllers can use it. - The volume controller uses `replica.spec.evictionRequested == true` in exactly the same way it previously used `replica.status.evictionRequested` to influence replica scheduling/deletion behavior (e.g. rebuild an extra replica to replace the evicting one or delete the evicting one once rebuilding is complete). - The node controller sets `replica.spec.evictionRequested = false` when: - The user is not requesting eviction with `node.spec.evictionRequested == true` or `disk.spec.evictionRequested == true`, AND - The conditions aren't right for auto-eviction based on the node status and drain policy. - The node controller sets `status.autoEvicting = true` when a node has evicting replicas because of the new drain policies and `status.autoEvicting == false` when it does not. This provides a clue to the user (and the UI) while auto eviction is ongoing. ### Test plan Test normal behavior with `block-for-eviction`: - Set `node-drain-policy` to `block-for-eviction`. - Create a volume. - Ensure (through soft anti-affinity, low replica count, and/or enough disks) that an evicted replica of the volume can be scheduled elsewhere. - Write data to the volume. - Drain a node one of the volume's replicas is scheduled to. - While the drain is ongoing: - Verify that the volume never becomes degraded. - Verify that `node.status.autoEvicting == true`. - Verify that `replica.spec.evictionRequested == true`. - Verify the drain completes. - Uncordon the node. - Verify the replica on the drained node has moved to a different one. - Verify that `node.status.autoEvicting == false`. - Verify that `replica.spec.evictionRequested == false`. - Verify the volume's data. - Verify the output of an appropriate event during the test (with reason `EvictionAutomatic`). Test normal behavior with `block-for-eviction-if-contains-last-replica`: - Set `node-drain-policy` to `block-for-eviction-if-contains-last-replica`. - Create one volume with a single replica and another volume with three replicas. - Ensure (through soft anti-affinity, low replica count, and/or enough disks) that evicted replicas of both volumes can be scheduled elsewhere. - Write data to the volumes. - Drain a node both volumes have a replica scheduled to. - While the drain is ongoing: - Verify that the volume with one replica never becomes degraded. - Verify that the volume with three replicas becomes degraded. - Verify that `node.status.autoEvicting == true`. - Verify that `replica.spec.evictionRequested == true` on the replica for the volume that only has one. - Verify that `replica.spec.evictionRequested == false` on the replica for the volume that has three. - Verify the drain completes. - Uncordon the node. - Verify the replica for the volume with one replica has moved to a different node. - Verify the replica for the volume with three replicas has not moved. - Verify that `node.status.autoEvicting == false`. - Verify that `replica.spec.evictionRequested == false` on all replicas. - Verify the the data in both volumes. - Verify the output of two appropriate events during the test (with reason `EvictionAutomatic`). - Verify the output of an appropriate event for the replica that ultimately wasn't evicted during the test (with reason `EvictionCanceled`). Test unschedulable behavior with `block-for-eviction`: - Set `node-drain-policy` to `block-for-eviction`. - Create a volume. - Ensure (through soft anti-affinity, high replica count, and/or not enough disks) that an evicted replica of the volume can not be scheduled elsewhere. - Write data to the volume. - Drain a node one of the volume's replicas is scheduled to. - While the drain is ongoing: - Verify that `node.status.autoEvicting == true`. - Verify that `replica.spec.evictionRequested == true`. - Verify the drain never completes. - Uncordon the node. - Verify that the volume is healthy. - Verify that `node.status.autoEvicting == false`. - Verify the volume's data. - Verify the output of an appropriate event during the test (with reason `EvictionAutomatic`). - Verify the output of an appropriate event during the test (with reason `EvictionCanceled`). ### Upgrade strategy - Add `status.autoEvicting = false` to all `node.longhorn.io` objects during the upgrade. - Add `spec.evictionRequested = status.evictionRequested` to all replica objects during the upgrade. - The default node drain policy remains `Block If Contains Last Replica`, so do not make setting changes. ## Note I have given some though to if/how this behavior should be reflected in the UI. In this draft, I have [chosen not to represent auto-eviction as a node condition](#api-changes), which would have automatically shown it in the UI, but awkwardly. I considered representing it in the `Status` column on the `Node` tab. Currently, the only status are `Schedulable` (green), `Unschedulable` (yellow), `Down` (grey), and `Disabled` (red). We could add `AutoEvicting` (yellow), but it would overlap with `Unschedulable`. This might be acceptable, as it could be read as, "This node is auto-evicting in addition to being unschedulable." ================================================ FILE: enhancements/20230905-snapshot-space-management.md ================================================ # Snapshot Space Management ## Summary This feature allows users to control the count and size of snapshots of a volume. ### Related Issues https://github.com/longhorn/longhorn/issues/6563 ## Motivation ### Goals - A replica honors snapshot count and size limitations when it creates a new snapshot. - Users can set snapshot count and size limitation on a Volume. - Users can set a global default snapshot maximum count. ### Non-goals [optional] - Snapshot space management on a node or disk level. - Freeze disk before taking snapshot. - Auto deleting snapshots if there is no space for creating a new system snapshot. ## Proposal ### User Stories With snapshot space management, snapshot space usage is controllable. Users can set snapshot limitation on a Volume and evaluate maximum space usage. ### User Experience In Detail Before snapshot space management: The default maximum snapshot count for each volume is `250`. This is a constants value in the [source code](https://github.com/longhorn/longhorn-engine/blob/8b4c80ab174b4f454a992ff998b6cb1041faf63d/pkg/replica/replica.go#L33) and users don't have a way to control it. If a volume size is 1G, the maximum snapshot space usage will be 250G. After snapshot space management: There is a configurable default maximum snapshot count setting. Users can update it to overwrite the fixed value in the system. Users can set different maximum snapshot count and size on each volume. A more important volume can have more snapshot space usage. ## Design ### Implementation Overview #### Settings Add a new setting definition: ```go const ( SettingNameSnapshotMaxCount = SettingName("snapshot-max-count") ) var ( SettingDefinitionSnapshotMaxCount = SettingDefinition{ DisplayName: "Snapshot Maximum Count", Description: "Maximum snapshot count for a volume. The value should be between 2 to 250", Category: SettingCategorySnapshot, Type: SettingTypeInt, Required: true, ReadOnly: false, Default: "250", } ) ``` #### Volume CRD Add two fields to `VolumeSpec`: ```go type VolumeSpec struct { // ... // +optional SnapshotMaxCount int `json:"snapshotMaxCount"` // +kubebuilder:validation:Type=string // +optional SnapshotMaxSize int64 `json:"snapshotMaxSize,string"` } ``` - If `SnapshotMaxSize` is `0`, it means there is no snapshot size limit for a Volume. #### Volume mutator - If `SnapshotMaxCount` is `0`, using the `snapshot-max-count` setting value to update it. - If a volume is expanded, checking whether `snapshot-max-size` is smaller than `size * 2`. If it is, using `size * 2` to update it. #### Volume validator - The `SnapshotMaxCount` should be between `2` to `250`. This limitation includes user and system snapshot. In LH, all snapshots can be merged to one snapshot, so at least one snapshot can't be delete. To create another snapshot, we need to have enough count for it. In conclusion, the minimum value for `SnapshotMaxCount` is `2`. - If `SnapshotMaxSize` is't `0`. The minimum value for `SnapshotMaxSize` is same as `Size * 2` in a Volume, because a volume can have at least two snapshots. #### Replica - Add two fields to `Replica`: ```go type Replica struct { // ... snapshotMaxCount int snapshotMaxSize int64 } ``` - Add a function `GetSnapshotCountUsage` to retrieve snapshot count usage. We should skip the volume head, backing disk, and removed disks. - Add a function `GetSnapshotSizeUsage` to retrieve total snapshot size usage. We should skip the volume head, backing disk, and removed disks. #### ReplicaServer - Add the `remain_snapshot_size` field to `Replica` proto message: ```protobuf message Replica { // ... int64 remain_snapshot_size = 17; } ``` - Update the `getReplica` function to return `SnapshotCountUsage` and `SnapshotSizeUsage` fields. #### Remote backend - Add a `RemainSnapshotSize` field to `ReplicaInfo`: ```go type ReplicaInfo struct { // ... RemainSnapshotSize int `json:"remainsnapshotSize"` } ``` - Add a new function `GetSnapshotCountAndSizeUsage` to return current snapshot count and size usage. #### Replicator - Add a new function `GetSnapshotCountAndSizeUsage` to return current snapshot count and size usage. We should get the biggest value from all replicas, because replica data may be unsynced when the system is unsteady. #### Controller engine - Add a new function `canDoSnapshot` to check whether snapshot count and size usage is under limitation. #### Manager API - Add new action `updateSnapshotMaxCount` to `Volume` (`"/v1/volumes/{name}"`) - Add new action `updateSnapshotMaxSize` to `Volume` (`"/v1/volumes/{name}"`) ### Test plan Integration test plan. 1. `snapshot-max-count` setting - Validate the value should be between `2` to `250`. - Create a Volume with empty `SnapshotMaxCount` and mutator should replace the value with `snapshot-max-count` setting value. - Create a Volume with nonempty `SnapshotMaxCount` and mutator shouldn't update the value. 2. A volume with `1G` size, `2` snapshot max count, and `0` snapshot max size. - Create the first snapshot. It should be successful. - Create the second snapshot. It should be successful. - Create the third snapshot. It should be failed. - Delete a snapshot and create a new snapshot. It should be successful. 3. A volume with `1G` size, `250` snapshot max count, and `2.5G` snapshot max size. - Write `0.5G` data and create the first snapshot. It should be successful. - Write `1G` data and create the second snapshot. It should be successful. - Write `1G` data and create the third snapshot. It should be successful. - Write `1G` data and create the fourth snapshot. It should be failed. - Delete the second or third snapshot and create a new snapshot. It should be successful. ### Upgrade strategy `None` ================================================ FILE: enhancements/20231204-default-priority-class.md ================================================ # Default Priority Class ## Summary Pods can have priority which indicates the importance of a pod relative to other pods. Longhorn pods will face unexpected eviction due to insufficient cluster resources or being preempted by higher-priority pods A PriorityClass defines a mapping from a priority class name to the integer value of the priority, where a higher value signifies greater priority.. ### Related Issues [Have default priorityClass to prevent unexpected longhorn pods eviction.](https://github.com/longhorn/longhorn/issues/6528) ## Motivation ### Goals - Reduce the opportunity of the unexpected eviction of Longhorn pods. ### Non-goals [optional] `None` ## Proposal Have a default priority class with the highest value to all Longhorn pods. ### User Stories After a fresh installation or an upgrade, users will have Longhorn pods with the field `priorityClassName: longhorn-critical` in `spec` to prevent Longhorn pods from unexpected eviction. - For install, the priority class will not be set using the default value if the value has been provided by users. - For upgrade, the priority class will not be set using the default value if the settings already exists. ### User Experience In Detail 1. For a fresh install, the default priority class will be set if there is no priority class set by users. The setting of the priority class will be updated. 2. For an upgrade, if all volumes are detached and the setting of the priority class is empty, the setting should be applied including user-managed components. The setting of the priority class will be updated because the source of truth of settings is the config map (longhorn-default-setting). 3. For an upgrade, no matter weather all volumes are detached or not, if the setting of the priority class has been set, the setting should not be updated with the default priority class. 4. For an upgrade, if volumes are attached, the setting should be applied to the user-managed components only if they haven't been set. The setting of the priority class will not be updated. NOTE: Before modifying the setting `priority-class`, all Longhorn volumes must be detached. ### API changes `None` ## Design ### Implementation Overview 1. Adding a new template YAML file in the Longhorn chart/templates to create a default priority class. ```yaml apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: longhorn-critical # The example value of a production is 4000000 from https://kubernetes.io/blog/2023/01/12/protect-mission-critical-pods-priorityclass # Most of the paragraph uses 1000000 for high priority class. # system default priority classes: # """txt # NAME VALUE GLOBAL-DEFAULT AGE # system-node-critical 2000001000 false 5h30m # system-cluster-critical 2000000000 false 5h30m # """ # The maximum allowed value of a user defined priority is 1000000000 value: 1000000000 description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure" globalDefault: false preemptionPolicy: PreemptLowerPriority ``` 2. Set the `defaultSettings.priorityClass` to `longhorn-critical` and `priorityClass` of Longhorn components in the value.yaml of Longhorn chart. ```yaml ... defaultSettings: ... priorityClass: &defaultPriorityClassNameRef "longhorn-critical" ... longhornManager: log: # -- Options: `plain`, `json` format: plain # -- Priority class for longhorn manager priorityClass: *defaultPriorityClassNameRef longhornDriver: # -- Priority class for longhorn driver priorityClass: *defaultPriorityClassNameRef longhornUI: # -- Priority class count for longhorn ui priorityClass: *defaultPriorityClassNameRef ``` ### Test plan 1. Modify the test case `test_setting_priority_class` to check if there is a default priority class `longhorn-critical` existing. 2. Add a test case where the `defaultSettings.priorityClass` has been modified by users and the setting `priority-class` should be the same to the value provided by users after a fresh install. 3. Add a test case where the setting `priority-class` will be updated if all volumes are detached and the setting of the priority class is not set for an upgrade. 4. Add a test case where the setting `priority-class` will not be updated after the upgrade if any volume is attached. 5. Add a test case where the setting `priority-class` will not be updated if the setting of the priority class has been set for an upgrade. ### Upgrade strategy When adding a priority class, all Longhorn volumes must be detached. Therefore we will not change the setting `priority-class` if there is any volume attached to the node or the setting `priority-class` is set to a priority class. ## Note [optional] `None` ================================================ FILE: enhancements/20231226-support-non-disruptive-volume-related-setting-updates.md ================================================ # Support non-disruptive volume-related setting updates ## Summary The volume-related settings can only be updated when no volumes are attached. This enhancement will allow users to modify volume-related settings, with changes taking effect only during volume upgrades or reattachments. ### Danger Zone If modifying the settings that will affect the running volumes (usually it will restart instance manager pods, and restarting instance manager pods will affect the running volumes), the setting should be included in the Danger Zone settings. | Setting | Additional Information| Affected Components | | --- | --- | --- | | [Kubernetes Taint Toleration](#kubernetes-taint-toleration)| [Taints and Tolerations](../../advanced-resources/deploy/taint-toleration/) | System-managed components | | [Priority Class](#priority-class) | [Priority Class](../../advanced-resources/deploy/priority-class/) | System-managed components | | [System Managed Components Node Selector](#system-managed-components-node-selector) | [Node Selector](../../advanced-resources/deploy/node-selector/) | System-managed components | | [Storage Network](#storage-network) | [Storage Network](../../advanced-resources/deploy/storage-network/) | Instance Manager and Backing Image components | | [V1 Data Engine](#v1-data-engine) || Instance Manager component | | [V2 Data Engine](#v2-data-engine) | [V2 Data Engine (Preview Feature)](../../v2-data-engine/) | Instance Manager component | | [Guaranteed Instance Manager CPU](#guaranteed-instance-manager-cpu) || Instance Manager component | | [Guaranteed Instance Manager CPU for V2 Data Engine](#guaranteed-instance-manager-cpu-for-v2-data-engine) || Instance Manager component | ### Related Issues https://github.com/longhorn/longhorn/issues/7173 ## Motivation Longhorn must have the capability to update volume-related settings without deleting pods associated with running volumes. Without this enhancement, users are restricted to customizing the volume-related settings only when all volumes are detached. ### Goals - Users can update the volume-related settings immediately. - The updated volume-related settings will take effect in Longhorn only during volume upgrades or reattachments. - A field `Applied` in `Status` of the CRD `Setting` indicating that the setting has been applied to the Longhorn components. ### Non-goals [optional] `None` ## Proposal This proposal aims to enable users to update the volume-related settings while ensuring the uninterrupted operation of running volumes. ### User Stories Previously, users could only update volume-related settings after manually detaching all volumes. With this enhancement, users can promptly update the settings, and Longhorn will apply the change to an instance manager only when there are no engine or replica processes running on that instance manager. ### User Experience In Detail Users can modify the volume-related settings, like other general settings, either through the Longhorn UI or by applying the Kubernetes manifest file. Longhorn carefully examine these settings to determine whether to proceed with the update process. If there are engine or replica processes running on the instance manager, Longhorn will temporarily halt the update process and retry until all processes of the instance manager are terminated. Longhorn components, except the instance manager, must still wait for all volumes to be detached before applying the setting. Once all volumes are detached, users may need to reapply the setting to ensure it affects the other components. #### Check Setting Status By CLI Users can check if the setting has been applied to Longhorn components by the CLI command: ```shell > kubectl -n longhorn-system get setting priority-class NAME VALUE APPLIED AGE priority-class longhorn-critical true 5h30m ``` #### Check Setting Status By UI Users can check if the setting has been applied to Longhorn components on the Setting/General page. The Danger Zone section will display any unapplied Danger Zone settings. ![](./image/danger_zone_unapplied_setting.png) ### API changes Add a field `Applied` in returning data `Setting` for APIs `SettingList`, `SettingGET` and `SettingSet`: ```golang type Setting struct { ... Applied bool `json:"applied"` Name string `json:"name"` Value string `json:"value"` ... } ``` ## Design ### CRD ```golang type Setting struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` // The value of the setting. Value string `json:"value"` // The status of the setting. Status SettingStatus `json:"status,omitempty"` } // SettingStatus defines the observed state of the Longhorn setting type SettingStatus struct { // The setting is applied. Applied bool `json:"applied"` } ``` ### Implementation Overview 1. Move `SettingNameV1DataEngine`, `SettingNameV2DataEngine`, and `SettingNameV2DataEngineGuaranteedInstanceManagerCPU` settings to the Category `SettingCategoryDangerZone`. 1. Remove all volumes attached examinations for volume-related settings from validations. Such as: ```golang case types.SettingNameTaintToleration: volumesDetached, err := s.AreAllVolumesDetachedState() if err != nil { return errors.Wrapf(err, "failed to list volumes before modifying toleration setting") } if !volumesDetached { return &types.ErrorInvalidState{Reason: "cannot modify toleration setting before all volumes are detached"} } ``` 1. Separate the non-danger zone settings and danger zone setting in the setting controller. ```golang func (sc *SettingController) handleErr(err error, key interface{}) { ... _, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { return err } err := sc.syncNonDangerZoneSettingsForManagedComponents(name) return sc.syncDangerZoneSettingsForManagedComponents(name) } ``` 1. Block applying the settings to system-managed components if one or more volume are attached: For settings: ```golang dangerSettingsRequiringAllVolumesDetached := []types.SettingName{ types.SettingNameTaintToleration, types.SettingNameSystemManagedComponentsNodeSelector, types.SettingNamePriorityClass, types.SettingNameStorageNetwork, } ``` ```golang if slices.Contains(dangerSettingsRequiringAllVolumesDetached, settingName) { detached, _, err := sc.ds.AreAllVolumesDetached(longhorn.DataEngineTypeAll) ... if !detached { return &types.ErrorInvalidState{Reason: fmt.Sprintf("failed to apply %v setting to Longhorn components when there are attached volumes. It will be eventually applied", settingName)} } // apply the setting to components. ... } ``` 1. For the settings related to v1/v2 data engine volume, we will check if volumes are attached according to the data engine type. ```golang dangerSettingsRequiringSpecificDataEngineVolumesDetached := []types.SettingName{ types.SettingNameV1DataEngine, types.SettingNameV2DataEngine, types.SettingNameGuaranteedInstanceManagerCPU, types.SettingNameV2DataEngineGuaranteedInstanceManagerCPU, } ``` ```golang if slices.Contains(dangerSettingsRequiringSpecificDataEngineVolumesDetached, settingName) { switch settingName { case types.SettingNameV1DataEngine, types.SettingNameV2DataEngine: // the webhook validators will check if volumes are attached according to the data engine type. case types.SettingNameGuaranteedInstanceManagerCPU, types.SettingNameV2DataEngineGuaranteedInstanceManagerCPU: detached, _, err := sc.ds.AreAllVolumesDetached(dataEngine) if !detached { return &types.ErrorInvalidState{Reason: fmt.Sprintf("failed to apply %v setting to Longhorn components when there are attached volumes. It will be eventually applied", settingName)} } // apply the setting to components. ... } } ``` 1. Manage the instance manager pods in the instance controller and restart pods if no instances are running and settings are not applied. ```golang func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) error { err := imc.annotateCASafeToEvict(im) if err != nil { return err } ... // check if pods of im *longhorn.InstanceManager needs to be restarted. isSettingSynced, isPodDeletedOrNotRunning, areInstancesRunningInPod, err := imc.areDangerZoneSettingsSyncedToIMPod(im) isPodSettingSynced := isSettingSynced || isPodDeletedOrNotRunning || areInstancesRunningInPod if im.Status.CurrentState != longhorn.InstanceManagerStateError && im.Status.CurrentState != longhorn.InstanceManagerStateStopped && isPodSettingSynced { return nil } // cleanup and recreate pods. ... } func (imc *InstanceManagerController) isSettingsSyncedToPod(im *longhorn.InstanceManager) (isSynced, isPodDeletedOrNotRunning, areInstancesRunningInPod bool, err error) { // check if instance manager is in running state. // check if instance manager has running instances. // check if the settings related to system managed components are applied to pods of the instance manager. return true, false, false err } ``` ### Test plan - Test no volume attached 1. Fresh install Longhorn 1. Customize a Danger Zone setting such as setting the value of `priority-class` to be `system-cluster-critical` 1. The setting applies to all components managed by Longhorn. 1. The `Status.Applied` of the setting is `true`. - Test a volume attached with a replica on three node cluster. 1. Fresh install Longhorn. 1. Create a volume with a replica. 1. Attach the volume to a node. 1. Customize a Danger Zone setting such as setting the value of `priority-class` to be `system-cluster-critical` 1. The setting only applies to pods of the instance manager without engine and replica processes. 1. The `Status.Applied` of the setting is `false`. 1. Detach the volume. 1. Reapply the setting. 1. The setting applies to all components managed by Longhorn. 1. The `Status.Applied` of the setting is `true`. ### Upgrade strategy - Add `spec.syncRequestedAt = now()` to all `settings.longhorn.io` objects during the upgrade. - Check if settings related to system components should be applied to components managed by Longhorn. ## Note [optional] `None` ================================================ FILE: enhancements/20240108-replica-auto-balance-pressured-disks.md ================================================ # Replica Auto-Balance Pressured Disks On Same Node ## Summary This proposal enhances Longhorn's replica auto-balancing feature to address disk space pressure caused by growing volumes. It introduces a new setting, `replica-auto-balance-disk-pressure-percentage`, allowing users to define a percentage threshold for automatic action. ### Related Issues https://github.com/longhorn/longhorn/issues/4105 ## Motivation ### Goals - **Reduced Manual Intervention:** Automatic rebalances replicas during disk pressure, minimizing manual effort. - **Improved Performance:** Potentially faster replica rebuilds within the same node using local file copy. ### Non-goals [optional] - This proposal does not address auto-balancing replicas across different nodes. Longhorn prioritizes balancing across nodes and zones before disks within a node. - Performance evaluation of local file copy is outside the scope of this proposal. ## Proposal ### User Stories #### Story 1: Reduced Manual Intervention - **As a Longhorn user:** I want Longhorn to automatically handle replica movement based on disk pressure, minimizing manual intervention. - **Before:** I need to regularly check disk space and manually trigger replica movement to address pressure situations. - **After:** Longhorn automatically rebalances replica when disk pressure threshold percentage is reached, freeing me up for other tasks. #### Story 2: Improved Replica Rebuild Performance - **As a Longhorn user:** I want Longhorn to leverage local file copy for replica rebuilding within the same node, potentially improving rebuild speed. - **Before:** Replica rebuild relies on TCP transfer, which can be slower than local file copy. - **After:** When rebuilding replicas on the same node, Longhorn uses local file copy for data transfer. ### User Experience In Detail 1. **Setup:** - You have a Longhorn cluster with multiple nodes, each containing multiple schedulable disks. - Configure via Longhorn UI or Kubernetes CRD. - Set `replica-auto-balance` to `best-effort`. - Define the `replica-auto-balance-disk-pressure-percentage` (e.g., 90% used space triggers disk rebalancing). > **Note:** This feature is not affected by the `replica-soft-anti-affinity` setting. This means that even if the node soft anti-affinity prevents Longhorn from rebuilding a replica on the same node, this feature will still attempt to rebuild a replica on the same node but on a different disk for migration purpose. 1. **Deploy Workload:** Deploy workload utilizing Longhorn volumes. 1. **Replica Disk Reaches Pressured Threshold:** - Longhorn automatically find another disk on the same node with more available disk space. - Longhorn rebuild replica on the same node using the local file copy for data transfer. ## Design ### New Setting: `replica-auto-balance-disk-pressure-percentage` This setting allows defining the disk pressure threshold (percentage of used space) that triggers automatic rebalancing. Its only functional when `replica-auto-balance` is set to `best-effort`. ```go SettingDefinitionReplicaAutoBalanceDiskPressurePercentage = SettingDefinition{ DisplayName: "Replica Auto Balance Disk Pressure Threshold (%)", Description: "Sets the threshold percentage of disk space utilization that triggers replica auto-balance.\n\n" + "When the threshold percentage is reached, Longhorn automatically rebuilds replicas under disk pressure onto another disk within the same node.\n\n" + "**Note:** This setting is effective only under the following conditions:\n" + "- **Replica Auto Balance** is set to **best-effort**.\n" + "- There must be at least one other disk on the node with sufficient available space.\n\n" + "To disable this feature for replica auto-balance (best-effort), set the value to 0.", Category: SettingCategoryScheduling, Type: SettingTypeInt, Required: true, ReadOnly: false, Default: "90", } ``` #### Balancing Replicas In Disk Pressure 1. **Auto-Balance Check** The volume controller checks if replicas are node and zone balanced and then verifies if any are under the disk pressure threshold percentage. ```golang func (rcs *ReplicaScheduler) IsDiskUnderPressure(diskPressurePercentage int64, info *DiskSchedulingInfo) bool { storageUnusedPercentage := int64(0) storageUnused := info.StorageAvailable - info.StorageReserved if storageUnused > 0 && info.StorageMaximum > 0 { storageUnusedPercentage = storageUnused * 100 / info.StorageMaximum } return storageUnusedPercentage < 100 - int64(diskPressurePercentage) } ``` 1. **Replica Candidate Selection:** The volume controller prioritizes replicas under pressure by selecting the first on from the sorted list of `node.Status.DiskStatus.ScheduledReplica`. This approach prevents overwhelming a single disk with all pressured replicas. 1. **Candidate Disk Search:** The controller searches for a suitable candidate disk on the same node. This disk must have sufficient available storage to accommodate the replica after the rebuild without exceeding the pressure threshold itself. ```go func (rcs *ReplicaScheduler) IsSchedulableToDiskConsiderDiskPressure(diskPressurePercentage, size, requiredStorage int64, info *DiskSchedulingInfo) bool { newDiskUsagePercentage := (requiredStorage + info.StorageScheduled + info.StorageReserved) * 100 / info.StorageMaximum logrus.WithFields(logrus.Fields{ "diskUUID": info.DiskUUID, "diskPressurePercentage": diskPressurePercentage, "requiredStorage": requiredStorage, "storageScheduled": info.StorageScheduled, "storageReserved": info.StorageReserved, "storageMaximum": info.StorageMaximum, }).Debugf("Evaluated new disk usage percentage after scheduling replica: %v%%", newDiskUsagePercentage) return rcs.IsSchedulableToDisk(size, requiredStorage, info) && newDiskUsagePercentage < int64(diskPressurePercentage) } ``` 1. **Replenish Replica With Local File Copy:** Once a candidate replica and disks are identified, the volume controller proceeds with the current replica replenish flow and leverage local file copy for replica rebuilding. ### Local Disk Replica Rebuild #### Longhorn Manager During replica rebuilding, if another replica exists on the same node, add the source and target file directories to the `rpc.EngineReplicaAddRequest.LocalSync` for local file copy. ```go localSync = &etypes.FileLocalSync{ Source: filepath.Join(nodeReplica.Spec.DiskPath, "replicas", nodeReplica.Spec.DataDirectoryName), Target: filepath.Join(targetReplica.Spec.DiskPath, "replicas", targetReplica.Spec.DataDirectoryName), } ``` #### Longhorn instance manager 1. Introduce `EngineReplicaLocalSync` to `EngineReplicaAddRequest` message in `proxy.proto`. ```go message EngineReplicaAddRequest { ProxyEngineRequest proxy_engine_request = 1; string replica_address = 2; bool restore = 3; int64 size = 4; int64 current_size = 5; bool fast_sync = 6; int32 file_sync_http_client_timeout = 7; string replica_name = 8; int64 grpc_timeout_seconds = 9; EngineReplicaLocalSync local_sync = 10; } message EngineReplicaLocalSync{ string source = 1; string target = 2; } ``` 1. Receive Local Sync Information: The proxy server received the `EngineReplicaAddRequest` containing the `FileLocalSync` message. 1. Forward Local Sync Information: The proxy server forwards it to the Longhorn engine's `FilesSyncRequest.FileLocalSync`. #### Longhorn Engine 1. Introduce `FileLocalsync` to `FileSyncRequest` message in `synagent.proto`. ```go message FilesSyncRequest { string from_address = 1; string to_host = 2; repeated SyncFileInfo sync_file_info_list = 3; bool fast_sync = 4; int32 file_sync_http_client_timeout = 5; FileLocalSync local_sync = 6; } message FileLocalSync{ string source = 1; string target = 2; } ``` 1. Identify Local Sync Request: The sync-agent received the `FileSyncRequest` containing the `FileLocalSync` message. 1. Perform Local File Copy: If the `FileLocalSync` message is provided, the sync-agent uses the provided source and target paths to directly copy the replica data within the same node. New functions will be implemented in https://github.com/longhorn/sparse-tools/ for synchronizing data contents using local files. 1. Fallback to TCP Transfer (if needed): In case of errors during local file copy, the sync-agent removed file created to the target directory and falls back to the approach of transferring data using TCP. ### Test Plan - Validate this feature alleviates disk pressure within the same node. - Validate this feature does not attempt to rebuild replica to another disk if the disk will fall below the pressure threshold percentage after replica is rebuilt on it. - Confirm this feature does not attempt to rebuild all replicas to a the same disk simultaneously, which could cause a rebuilding loop. ### Upgrade strategy This feature introduces a new setting and is non-disruptive to existing Longhorn system. User can continue using exiting volumes as before and the feature will automatically apply if the cluster have `replica-auto-balance` set to `best-effort` during the upgrade process. ### Limitations In the current Longhorn version, There can be a delay up to 30 seconds for the disk storage usage to reflect after a replica is removed from the disk. This can cause additional replica to be rebuilt before the node controller's disk monitor detects the space change. ================================================ FILE: enhancements/20240314-recurring-and-manual-full-backup-support.md ================================================ # 20240314-recurring-and-manual-full-backup-support ## Summary This feature enables Longhorn to create **recurring job** for full backup or **manually trigger** the full backup of the volume. ### Related Issues - Community issue: https://github.com/longhorn/longhorn/issues/7069 - Improvement issue: https://github.com/longhorn/longhorn/issues/7070 ## Motivation Longhorn always does incremental backup which only backup the newly updated blocks. There is a chance that the previous backup blocks on the backupstore are corrupted. In this case, users can not restore the volume anymore because Longhorn aborts the restoration when it finds those blocks have different checksum. ### Goals - Add new fields to `RecurringJob` - `Parameters` to `Spec`: - `full-backup-interval`: used in `RecurringJob - Backup Type` to execute full backup every N incremental backups (default to 0 for always incremental) - For example, if N is 5, then after 5 regular incremental backups, the job will perform full backup for the 6th backup - `ExecutionCount` to `Status`: - So the job knows when to run full backup if `full-backup-interval` is provided. - Add a new fields `BackupMode` to `Backup.Spec` - `BackupMode`: used in `Backup` CR to trigger the full backup (Options: `"full"`, `"incremental"`, default to `"incremental"` for always incremental) - When doing full backup, Longhorn will backup **all the current blocks** of the volume and **overwrite them** on the backupstore even if those blocks already exists on the backupstore. - Collect metrics of `newly upload data size` and `overwritten data size` for user to better understand the cost. - `newly upload data size`: the data size uploaded to the backupstore in this backup - `overwritten data size`: the data size uploaded to the backupstore and overwritten the exists block on the backupstore. ## Proposal ### User Stories ### User Experience In Detail #### Recurring Full Backup - Always 1. Create a `Backup` task type RecurringJob with the parameter `full-backup-interval: 0` and assign it to the volume ``` apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-full-backup-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} parameters: full-backup-interval: 0 name: recurring-full-backup-per-min retain: 0 task: backup ``` 2. The RecurringJob runs and fully backup the volume every time. #### Recurring Full Backup - Every N Incremental Backups 1. Create a `Backup` task type RecurringJob with the label `full-backup-interval: 5` and assign it to the volume ``` apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-full-backup-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} parameters: full-backup-interval: 5 name: recurring-full-backup-per-min retain: 0 task: backup ``` 2. The RecurringJob runs and fully backup the volume every 5 incremental backups. #### Manual Full Backup 1. When creating backup, users can check the checkbox `Full Backup: []` or assign `BackupMode: full` to the spec. 2. The backup will be full backup. 3. Maybe adjust the UI to make the process more simple ## Design ### Implementation Overview #### Metrics 1. Add two new fields `new upload data size`, `reupload data size` to the Backup Status #### UI 1. In **Volume Page** >> **Create Backup** , add a checkbox `Full Backup: []` - If it is checked, automatically add the parameters `backup-mode: full` to the request payload - For example: ``` HTTP/1.1 POST /v1/volumes/${VOLUME_NAME}?action=snapshotBackup Host: localhost:8080 Accept: application/json Content-Type: application/json Content-Length: 55 { "parameters": { "backup-mode": "full", }, "name": ${BACKUP_NAME}, } ``` 2. In **Recurring Jo** >> **Create Recurring Job**, add a new sector for user to fill in the parameters when the task is `Backup` related task. - Currently only support: - `full-backup-interval` 3. In **Backup** >> **${BackVOlume}**, add a new field `Backup Mode` - If it has the parameters `backup-mode: full`, show `full`, otherwise show `incremental` #### CRD 1. **BackupVolume**: 2. **Backup**: add a new fields `BackupMode` to the `Spec`. - `BackupMode`: `"full"` to trigger full backup. Default to `"incremental"` for incremental backup 3. **RecurringJob**: add a new fields `parameters` to `Spec` and `ExecutionCount` to `Status`. - `Spec.Parameters["full-backup-interval"]`: Only used in `Backup` related task. Execute full backup every N incremental backups. Default to 0 for always incremental - `Status.ExecutionCount` to record how many job have been executed for this recurring job. Backup CR Example ```yaml apiVersion: longhorn.io/v1beta2 kind: Backup metadata: name: backup-abcde1234 namespace: longhorn-system spec: snapshot: fake-snapshot backupMode: full ``` #### Backupstore 1. Need to pass `parameters` through the grpc function call chain. 2. In our implementation, if the Volume has `lastBackup`, we then always perform incremental Backup. 3. Now, if `backup-mode: full` exists in the parameters, - we then pretend the last Backup does not exist and force it to do the full Backup. - overwrites the block on the backupstore even it already exists. 4. store the `new upload data size`, `reupload data size` to the Backup Status. #### Webhook 1. Check the parameters to prevent from typo. 2. ReucrringJob currently only accept `full-backup-interval` 3. Backup currently only accept `backup-mode` ### Test plan #### Manually Full Backup 1. Create a Volume 4MB and fill in the content. 2. Create a Backup of the Volume. 3. Intentionally replace the content of the first block(2MB) on the backupstore 4. Restore the Volume, and will get error logs like below ``` [pvc-XXXXXX] time="XXXX" level=error msg="Backup data restore Error Found in Server[gzip: invalid checksum]" ``` 5. Create a full backup with the Spec `BackupMode: full` 6. Restore the backup, this time should work #### Recurring Job Full Backup 1. Create a Volume 4MB and fill in the content. 2. Create a Backup of the Volume. 3. Intentionally replace the content of the first block(2MB) on the backupstore 4. Restore the Volume, and will get error logs like below ``` [pvc-XXXXXX] time="XXXX" level=error msg="Backup data restore Error Found in Server[gzip: invalid checksum]" ``` 5. Create a `backup` task type RecurringJob with the parameter `full-backup-interval: 1` and assign it to the volume ``` apiVersion: longhorn.io/v1beta2 kind: RecurringJob metadata: name: recurring-full-backup-per-min namespace: longhorn-system spec: concurrency: 1 cron: '* * * * *' groups: [] labels: {} parameters: full-backup-interval: 1 name: recurring-full-backup-per-min retain: 0 task: backup ``` 6. Wait for the recurring job to be finished. 7. Restore the backup, this time should work #### DR Volume with Full Backup 1. Setup a backupstore. 2. Create a pod with a volume and wait for pod to start. 3. Write data to `/data/test1` inside the pod. 4. Create the 1st backup for the volume. 5. Create a DR volume based on the backup and wait for the init restoration complete. 6. Write more data to the original volume and get the md5sum. 7. Create a full backup recurring job. 8. Wait for the full backup job to be finished. 9. Activate the DR volume and check the md5sum is the same as the original volume. ### Upgrade strategy No need. ## Note [optional] None. ================================================ FILE: enhancements/20240325-nfs-server-ha.md ================================================ # Fast Failover for RWX Volume's NFS Server ## Summary In Longhorn, a ReadWriteMany volume (and at some point, a ReadOnlyMany volume) is handled by an NFS server located within the share-manager pod. It mounts an RWO volume in the usual way, and then provides export and locking capabilities on top of it, which multiple workload pods can mount and write safely. There is only one share-manager pod for a volume, so if the node on which the pod is running fails, the responsible controller must notice and start a new pod on another node. After it starts, client pods can re-connect and resume I/O. But first, they negotiate with the new server about what their state should be, and there is a grace period in which writes are refused while waiting for any other possible clients to show up. In a previous enhancement, [Dedicated Recovery Backend for RWX Volume's NFS Server](https://github.com/longhorn/longhorn/blob/master/enhancements/20220727-dedicated-recovery-backend-for-rwx-volume-nfs-server.md), Longhorn incorporated a recovery backend that can speed up that process and guarantee that all the former clients have been accounted for. But a node failure still takes minutes to recover from, as noted in the [Longhorn docs](https://longhorn.io/docs/1.6.0/high-availability/node-failure/). In this enhancement, a mechanism is proposed to quickly detect and respond to share-manager pod failure independently of the Kubernetes node failure sequence and timing. ### Related Issues [https://github.com/longhorn/longhorn/issues/6205](https://github.com/longhorn/longhorn/issues/6205) ## Motivation The main goal is to minimize downtime on RWX workloads. In the current code, failure of a container in the share-manager pod or the pod itself is quickly handled. It takes about 20 seconds to restore write access, of which about half is spent restarting the workload pods and letting them remount. Longhorn's recovery backend has the ability to shorten the grace period wait when all clients are accounted for. The big problem happens when the node with the share-manager pod fails or restarts. Just the time it takes for the node to go `NotReady` - in Kubernetes, typically about 45 seconds - is already too long an outage for share-manager failover. We intend to make the NFS client recovery time for a failure of the node running the share-manager pod to be just as quick as for simple pod failure. ### Goals - Implement a fast failover mechanism that is not gated by Kubernetes' relatively slow detection and handling of node failure. Recovery should be a matter of seconds rather than minutes. - It should not require a restart of workload pods. - Operation of the recovery backend will be unchanged. - After the failure of the share-manager pod, the application on the client side’s IO operations will be stuck until the share-manager and NFS are recreated, and the NFS server grace period has elapsed (if there was a workload pod on the failed node). The interruption will be brief, with the NFS server back up within 20 seconds, and an overall outage of less than a minute. - Changing Kubernetes behavior via `node-monitor-period` and `node-monitor-grace-period` values in the Kubelet should make no difference to RWX failover. The time to accomplish node NotReady processing and return to Ready state should be independent of RWX HA time. ### Non-goals Duplicate NFS servers running simultaneously, providing Active/Active or Active/Passive redundancy. "True HA" with zero recovery time is a long-term desire but is extremely difficult to accomplish with NFS. See the section on [Alternatives](#alternatives) below. ## Proposal Fast failover needs several pieces. 1. Quick detection of failure Use the Kubernetes `Lease` mechanism. When an RWX volume is created, create the lease at the same time as the share-manager pod and service. Just like the service, it will have the same name and lifetime as the volume itself. The lease holder is not specified at the time of creation; it will be assigned when the pod is scheduled to a node and begins to run. This mimics Kubernetes' own internal mechanism for tracking node status, which is also lease-based. Detection of an unresponsive server could also be done with a monitor that makes a recurring call to the server pod, but the lease has the advantage that it reflects the server's ability to make an API call, rather than the connectivity from an arbitrarily chosen monitor node. Longhorn will - Update the lease periodically from ShareManager pod. - Use a goroutine in the share-manager controller to schedule a check of the lease for expiration. 2. Quick removal of the failed/failing server If the lease is expired, we infer that the pod's node is dead and another controller needs to take over. All other nodes will detect that. Longhorn needs to pick one, and establish that choice so that multiple controllers don't all attempt to drive the recovery. Longhorn will use the controller's `isResponsibleFor` method: - If the lease is not expired, move on to normal handling. - If it has, set the share-manager CR `Status.OwnerID` to this node, if another node hasn't already done so. (We can tell by comparing the owner field to the lease holder.) The new owner will act as an interim manager to do tear-down and recreation of the share manager pod. It updates the CR and, if successful, then returns "true". - The new owner is logged, and then the reconcile can continue. Mark the share-manager CR as error state and let it clean up the pod. - In cleanup, - Remember the old lease holder. - Clear the lease. - Delete the pod. - If the node is down *OR* expired, do a force delete of the share-manager pod as well. If it truly is down, this is the same as current code. If it is not, then the first delete should have succeeded, but either way, even if the pod is still running, this will ensure that Kubernetes does not route service traffic to it. Also, deletion allows reuse of the same standard pod name, avoiding one potential upgrade complication. - Other resources need to change ownership similarly. Modify the usual rules to capture our suspicion that the expired node is dead: - Keep track of the delinquent state in the volume's lease object (see `Design` section below). The otherwise informational Spec.AcquireTime field can be used for that. - Change `IsNodeDownOrDeleted` to check `IsNodeDownOrDeletedOrDelinquent` for other resources related to that volume, as well as any resource-specific `isResponsibleFor` calls. 3. Quick creation of a replacement This is already a normal part of the Share Manager lifecycle. But it does need an adjustment to avoid the former host, which Kubernetes will try to re-schedule to since Kubelet does not yet report it as down. - If the volume is delinquent, avoid its node (the current delinquent lease holder). Create the pod with the existing set of selectors, tolerations, and affinities defined for share-manager pods, with an extra added one-time anti-affinity for the delinquent node. Note that when the pod is scheduled, it might not be on the interim controller node, so ownership might change again. - After the pod is created, it will not clear the delinquent condition until the lease is taken over. That happens after the pod is running, so the deliquent state can still be checked for attachment actions, which must be satisfied for the NFS server to run. 4. Avoid client workload pod restart Longhorn currently restarts the workload pods in volume controller when the share-manager pod restarts. There does not appear to be any need to do so. If the server restarts beneath a client, it will reconnect and resume operation without having to restart the workload pod to kill the client process. If the outage is long enough, a client mounted with "soft" option may return an EIO or an ETIMEDOUT to its application. 5. Shorten the recovery grace period After a restart, the NFS server will wait for clients to re-establish connection and supply their state. The recovery grace period is usually configured to 90 seconds. If all workload pods are alive, they will attempt to restore a connection, and the recovery backend can terminate the wait for clients as soon as all are accounted for. But if a workload pod was on the failed node, it will not be able to clear its state. For the purposes of fast failover, Longhorn will configure the NFS server with a shorter grace period of only 30 seconds. This is a trade-off between speed and probability of a legitimate client being delayed in re-connecting. ### User Experience The changes should not alter the use of RWX volumes. There will be no observable difference until a node failure occurs (for whatever reason - Kubernetes upgrade, eviction, injected error). This is an `experimental` feature, and needs some soak time before it is the default. There will be a setting to enable or disable it, and it will be off in the first release. ## Design This feature uses two close but separate notions of pod state, "stale" and "delinquent". A pod is stale when its lease has expired - that is, not been renewed for some period longer than the renewal interval. The inference is that its node has failed, and it needs a controller on another node to take over ownership and do the work of deleting and recreating the pod. All nodes will monitor leases, and the first one to notice the stale state will modify the share-manager CR status to make itself the owner. When that has happened, the stale state can be cleared by zeroing the `RenewalTime` field of the lease. Other controllers will know that it is being handled. After a new node has claimed the interim ownership, it marks the lease as "delinquent" and that state is used so that the controller can know how to do recovery, and in particular to avoid sending any commands to the dead node. They would time out without having any effect. There is not actually a "delinquent" field in the lease, so the `AcquireTime` field is overloaded and set to zero when the current holder is presumed dead. The flow is laid out in this state diagram: ![state diagram](./assets/images/rwx-fast-failover/lease_cr_statemachine.png) The "delinquent" state remains until the lease is acquired by a new lease-holder node, which happens after - The old pod has been deleted, - A new pod created and scheduled, - The old pod's attachment is cleared and the new pod attached to its node, - The exported filesystem is mounted and the NFS server is running. Typically, the delinquent state is used to bypass communication to the node that would be allowed by Kubernetes, but is likely to time out. To keep that state from lasting indefinitely, and possibly blocking some recovery action in turn blocks the pod creation, the state can be cleared when the node in question actually does shift to `NotReady` state. At that point, fast failover is moot, but failover will revert to the normal sequence. ### Implementation Overview - **share-manager** Add a constant, `share-manager-renew-interval`. It should be often enough to detect failure quickly, but not so often that it overwhelms the network and API server with traffic. Hard-code this to 3 seconds. Add a goroutine to keep the lease from expiring. Every `share-manager-renew-interval`, get the lease for the RWX volume (name it the same as the volume name) and update its `lease.spec.renewTime` to "now". If the lease cannot be found, take no action. - **longhorn-manager** - setting Add a Boolean setting, `rwx-volume-fast-failover`. Because this is an experimental feature to begin with, default it to false. - share_manager_controller Add code to create a lease at the same time as the service. Set `lease.spec.leaseDurationSeconds` to a little over two lease renewal periods. At creation, also set `lease.spec.acquireTime` and `lease.spec.renewTime` to "now". Every sync, check the lease for each share-manager. If acquireTime == renewTime, assume that the share-manager image does not know to update the lease, so take no action. Otherwise, if status is StateRunning and renewTime + timeout is less than "now", mark the share-manager CR as `Error` to drive deletion and re-creation. Add logic to `isResponsibleFor` to check for staleness first and take over ownership. Modify the lease to mark its status on the node as delinquent. Remove the selector labels for admission webhook and recovery backend from the delinquent node's longhorn-manager pod. Add delinquency check to `cleanupShareManagerPod()` when deciding to force-delete. Add an anti-affinity to pod manifest creation to avoid the previous owner node. - node_controller Add code at startup to replace the selector labels for any services that may have been turned off while the node was delinquent. - datastore/longhorn Revise `IsNodeDownOrDeleted` to add an `IsNodeDownOrDeletedOrDelinquent` function that also takes the volume as a parameter, and check the lease's state. - volume_controller Add logic in `ReconcileShareManagerState()` to check the volume's storageclass for NFS mount options. If `parameters.nfsOptions` contains "hard", then request remount to force a restart as in current code. Otherwise, skip that (but make a similar event). This code is aware that the default option is not to hard mount NFS. - **nfs-ganesha (user-space NFS server)** No changes. ### Proof of Concept A lot of trial failovers have been executed, to test various aspects of the implementation. We have run with the changes listed, but using a "global" application of delinquency check with a condition set for the node itself. Running on a 3-worker node cluster with just a single, soft-mounted RWX volume (nginx RWX example) and forcing a node restart with `shutdown -r now` on the node console, Longhorn could destroy and re-create the share-manager pod in 16 seconds, as shown by the time for the new owner to take over and resume refreshing the lease. Note that for the lease to be refreshed, the pod must be running, implying that the volume is mounted. The workload pods themselves failed to be able to write for about 2 minutes, but most of that time was waiting for the client grace period to elapse. ### Risks and Mitigation - **Something goes not as expected** For all code paths, the fallback is to resort to the normal "not ready" path. So at worst, the failover time is the same as previous releases. Every effort should be made to ensure that the logging is clear and sufficient to tell why an expedited failover was not accomplished in such cases. - **Possible delay due to image pull on the successor node** This is taken care of by the added feature to pre-pull the share-manager pod image: [[IMPROVEMENT] Pre-pull images (share-manager image and instance-manager image) on each Longhorn node](https://github.com/longhorn/longhorn/issues/8376). - **Insufficient resources on any successor node** We rely on Kubernetes scheduling to pick an appropriate successor node that meets all the usual constraints, if one exists. If the pod can't be scheduled, there is little to be done immediately but to wait for the failed node to return. For the user, the solution might be to add resources or decrease the cluster load in order to allow space for failover scheduling. This design does not schedule the successor in advance (see [Active/Passive](#active-passive) below for notes on how that might work). - **False positives** The initial choice for timeout interval may prove to be too sensitive. It is not currently adjustable. Increasing the timeout will make it less likely to restart share-manager unnecessarily, but also increase the time to recovery. One open question is whether delays in lease renewal and false positives are more likely in a heavily loaded cluster. That can probably only be resolved with copious testing, including high-load cases. - **Webhook availability** Testing with the PoC uncovered a snag. Often, the share manager controller will get a failure when it updates the volume attachment. The error is a timeout while trying to call the admission webhook. That repeats until the failed node actually goes to "not ready", and then it succeeds, usually about 20-30 seconds later. It doesn't break anything, but it means that the failover takes as long as it would without this feature. In analysis with team members, we concluded that it is because the admission webhook is a service and one node of the cluster is picked to respond to the service IP address for any given request. It can happen that it is the same node that hosted the share manager and has failed. If so, calls to the webhook will time out until control is passed by Kubernetes to another node, which happens on its own timetable. On the PoC test cluster of three worker nodes, that's about 1/3 of the time. That means that failovers will intermittently and unpredictably take longer than they should. We can avoid this by making a label on the longhorn-manager pod specific to each webhook and using that label as the webhook's selector. If the node goes delinquent, remove the label to take that node's IP address out of the webhook service's endpoint slice. That will prevent the unresponsive node from being selected. The same applies to the recovery backend service as well. ### Upgrade Strategy There are two changes that need to be handled in an upgrade: - Creation of a new setting to define and default: `rwx-volume-fast-failover` (default, false). - Revision of webhook and recovery backend services to use dedicated labels as selectors. Both can be done in the upgrade manifest. After an upgrade, the share-manager pod would have to be restarted into the new share-manager image for lease management to work. That could happen as part of the upgrade, or as nodes are restarted for other reasons. The first restart would use the old, slow mechanism, but subsequent ones would use the new one. The logic for creation and checking of Lease records ensures that all parties know and implement the lease strategy. If some component doesn't, the behavior defaults to the same as current code. > Old longhorn-manager + new share-manager: No lease is created, so the share-manager just skips the renewal loop. > New longhorn-manager + old share-manager: Lease is created, but never claimed by a holder or renewed to advance the expiration time. Staleness check sees that there is no holder, so the lease is never expired. Failover is only based on reported K8s node state. ### Test Plan The Test Plan is similar to [Dedicated Recovery Backend for RWX Volume's NFS Server](https://github.com/longhorn/longhorn/blob/master/enhancements/20220727-dedicated-recovery-backend-for-rwx-volume-nfs-server.md). - Setup 3 worker nodes for the Longhorn cluster. For each test repeat once with default NFS mount options, including "soft" or "softerr", and once with custom options including "hard" mount. - Tests 1. Enable the feature. Create 1 RWX volume and then run an app pod with the RWX volume on each worker node. Execute the command in each app pod `( exec 7<>/data/testfile-${i}; flock -x 7; while date | dd conv=fsync >&7 ; do sleep 1; done )` where ${i} is the node number. Turn off or restart the node where share-manager is running. Once the share-manager pod is recreated on a different node, check - Expect - In the client side, IO to the RWX volume will hang until a share-manager pod replacement is successfully created on another node. - During the outage, the server rejects READ and WRITE operations and non-reclaim locking requests (i.e., other LOCK and OPEN operations) with an error of NFS4ERR_GRACE. - New share-manager pod is created in under 20 seconds. - Outage, including grace period, should be less than 60 seconds. - If locks cannot be reclaimed after a grace period, the locks are discarded and return IO errors to the client. The client reestablishes a new lock. 2. Turn the deployment into a daemonset in [example]([https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml](https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml) ) and disable `Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly`. Then, deploy the daemonset with a RWX volume. Turn off the node where share-manager is running. Once the share-manager pod is recreated on a different node, check - Expect - The other active clients should not run into stale handle errors after the failover. - New share-manager pod is created in under 20 seconds. - Outage, including grace period, should be less than 60 seconds. 3. Run a scale test. Use the deployment in [example]([https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml](https://github.com/longhorn/longhorn/blob/master/examples/rwx/rwx-nginx-deployment.yaml) ). Make as many instances of the deployment as possible, but at least 100. They should be evenly distributed across the worker nodes. Run once without the fast-failover feature enabled. Note the CPU use of longhorn-manager pods. Re-run with the feature on, and verify that the CPU use is not significantly larger. The load of lease-checking should be minimal. Turn off one of the nodes. That should cause about 1/3 of the pods to relocate. - Expect - All pods should be recreated within 20 seconds. - Outage, including grace period, for all pods should be less than 60 seconds. - RWX volumes not originally on the failed node should be unaffected. ## Alternatives In the LEP for the recovery backend, it said, (bullets added for emphasis) > To support NFS server's failover capability, we need to change both the client and server configurations. A dedicated recovery backend for Kubernetes and Longhorn is also necessary. > In the implementation, we will not implement the active/active or active/passive server pattern: > - Longhorn currently supports local filesystems such as ext4 and xfs. Thus, any change in the node, which is providing service, cannot update to the standby node. The limitation will hinder the active/active design. > - Currently, the creation of an engine process needs at least one replica and then exports the iSCSI frontend. That is, the standby engine process of the active/passive configuration is not allowable in current Longhorn architecture. Those factors are still present. But here are some possibilities that have been considered. ### Active/Active with no interruption. In order to do a more immediate failover, the [NFS-ganesha Wiki](https://github.com/nfs-ganesha/nfs-ganesha/wiki/NFS-Ganesha-and-High-Availability) advises > NFS-Ganesha does not provide its own clustering support, but HA can be achieved using Linux HA and points to several options for a Linux node clustering solution layered below ganesha, including Gluster or Ceph. Some of the difficulties are discussed in the nfs-ganesha issue [Does the nfs client can do auto failover to another good nfs-ganesha server if one ganesha server is down.](https://github.com/nfs-ganesha/nfs-ganesha/issues/761). There was a [proposal](https://www.snia.org/sites/default/files/Poornima_NFS_GaneshaForClusteredNAS.pdf) to extend nfs-ganesha with a Cluster Manager Abstraction Layer (CMAL) for the purpose, but it was abandoned. Here are some more recent solutions for "NFS clusters". - SUSE SLE 15 [Highly Available NFS Storage with DRBD and Pacemaker](https://documentation.suse.com/sle-ha/15-SP5/html/SLE-HA-all/article-nfs-storage.html) - Ubuntu [HighlyAvailableNFS](https://help.ubuntu.com/community/HighlyAvailableNFS) - Highly Available NFS based Kerberos KDC [Ganesha + GlusterFS + HAProxy](https://www.loadbalancer.org/blog/highly-available-shared-nfs-server/) - A video on [How to create a HA NFS Cluster using Pacemaker, Corosync, & DRBD on RHEL / AlmaLinux 9](https://www.youtube.com/watch?v=IxFI0Ms0ULA). Setup is far from trivial. All of these methods use a distributed filesystem or block driver to keep the metadata synchronized between the HA cluster nodes. They set up network access with a tool such as HAProxy to an IP address that will be handled by whichever node or nodes are alive, and use a tight synchronizer between the Linux server nodes. In a Kubernetes setting, that would become a synchronizer between the NFS server containers in the paired share-manager pods. That might be possible, but it would add a significant amount of configuration to the container, and traffic to the management network per volume. ### Active/Active as Load Balancer - [cephNFS](https://rook.io/docs/rook/latest/CRDs/ceph-nfs-crd/#example) can configure multiple active NFS servers, but it does not work quite the same way. Clients connect to either server, but it relies on "sticky" client connection to stay with the same server. It recommends setting active count to 1; otherwise a failover may block I/O for the grace period while clients move. - rook/ceph had a [ticket for the HA feature](https://github.com/rook/rook/issues/11526) which has been closed for lack of activity. ### Active/Passive Also termed `Active/Warm Standby`. There's a good discussion of the comparison in [this S3GW doc](https://github.com/s3gw-tech/s3gw/blob/main/docs/research/ha/RATIONALE.md). The share-manager pod is the Longhorn SPOF analog of S3GW's `radosgw`. Even though the feature has been de-prioritized, there is a residual Longhorn issue that still applies: [[FEATURE] Improving recovery times for non-graceful node failures #6803](https://github.com/longhorn/longhorn/issues/6803). In this situation, the implementation could look something like this: 1. Make the share-manager pod a Deployment of two pods. Force them via hard anti-affinity to locate on different nodes. (This might take some adjustment on single-node systems, but in that case, HA for node failure is impossible anyway. Still, Longhorn would need to alter the deployment to 1 node to avoid an ever-present unshedulable pod.) 2. Pick a leader pod, using [leader election](https://kubernetes.io/docs/concepts/architecture/leases/#leader-election) just as Longhorn upgrade package does. That leader can mount the volume and export it under the service's ClusterIP, just as at present. 3. Weight or configure the ClusterIP mapping so that all traffic to that address is routed to the leader. One way to do this is to prevent the non-leader from reporting as "ready" in its readiness probe. As mentioned, there are complications with the engine and attachments for the passive pod. There would be a lot of changes to tolerate a second engine and ensure that it is not used until failover. To attach the backup node, the mount has to be `AccessMode: ReadWriteMany` itself so that Kubernetes will permit it. It won't actually ever be simultaneously written, since there will be no NFS client traffic to it. It is more like the RWX mount used for migration. ### Active/Fast-Failover Referred to as "Active/Standby" in the S3GW discussion. This document will use the term "Fast-failover" to emphasize that the pipeline is rebuilt entirely at the time of failure. The focus of the implementation is to make the rebuild as quick as possible. This is the option proposed. Other options considered for quick failure detection of the share-manager pod, but discarded: - Apply a liveness probe to the share-manager pod. But that is probed from `kubelet` on the same node, so that is ineffective when the node itself is down. - Similar to [rook](https://github.com/rook/rook/pull/12845), implement a "ping" RPC call in share-manager to check that it is responsive. But then, what entity pings and decides it is unresponsive? One idea was let all worker nodes (longhorn-managers) ping all RWX volumes, but that doesn't scale very well as volumes and nodes increase in number. ## Other References #### NFSv4 implementation - [Network File System (NFS) Version 4 Protocol](https://datatracker.ietf.org/doc/html/rfc7530) - [Client recovery in NFS Version 4](https://docs.oracle.com/cd/E19120-01/open.solaris/819-1634/6n3vrg2al/index.html) - [Long client timeouts when failing over the NFS Ganesha IP resource](https://www.suse.com/support/kb/doc/?id=000019374) - [Necessary NFS Server Cluster Design for NFS Client Lock Preservation](https://www.suse.com/support/kb/doc/?id=000020396) - [How NFSv4 file delegations work](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-DE6FECB5-FA4D-4957-BA68-4B8822EF8B43.html) #### Nfs-ganesha - https://github.com/nfs-ganesha/nfs-ganesha/wiki/NFS-Ganesha-and-High-Availability - https://www.snia.org/sites/default/files/Poornima_NFS_GaneshaForClusteredNAS.pdf - https://lists.nfs-ganesha.org/archives/list/devel@lists.nfs-ganesha.org/thread/MLI3DRZ5MR5MC4GBREO5OR2Q2SXYK47V/ - https://github.com/nfs-ganesha/nfs-ganesha/issues/761 #### Kubernetes - [K8s non-graceful node shutdown](https://kubernetes.io/blog/2023/08/16/kubernetes-1-28-non-graceful-node-shutdown-ga) - Poison-pill HA demo at [medik8s](https://www.medik8s.io) - Comparatively slow and focus is on ensuring against split-brain, not speed of recovery. ================================================ FILE: enhancements/20240402-share-manager-scheduling.md ================================================ # Share Manager Scheduling ## Summary In the Longhorn storage system, a share manager pod of an RWX volume is created on a random node, without the ability for users to specify a preferred locality. The purpose of this feature is to enhance the locality of an RWX volume and its share manager pod. ### Related Issues https://github.com/longhorn/longhorn/issues/7872 https://github.com/longhorn/longhorn/issues/4863 https://github.com/longhorn/longhorn/issues/8255 https://github.com/longhorn/longhorn/issues/2335 ## Motivation ### Goals - Share manager pod respects the node selector specified in `storageClass.parameters["shareManagerNodeSelector"]` - Share manager pod complies with the affinity rules defined in `storageClass.allowedTopologies` - Share manager pod respects the newly introduced `storageClass.Parameters["shareManagerTolerations"]` ### Non-goals [optional] `None` ## Proposal ### User Stories A share manager pod of an RWX volume is unable to adhere to the specified node selector and affinity rules, leading to potential inefficiencies and performance issues. The feature aims to enhance this by ensuring share manager pods can be scheduled according to the given node selector and affinity rules. ### User Experience In Detail The introduction of node selector and affinity rule capabilities in a storage class significantly enhances user control over the scheduling of share manager pods for RWX volumes. - Node Selector: Users can define node selectors directly within the storage class configuration by setting `storageClass.parameters["shareManagerNodeSelector"]`. When a share manager pod is to be scheduled, Kubernetes evaluates the node selectors specified by the user. The scheduler then ensures that the pod is placed only on nodes that match all of the specified labels. This mechanism provides a straightforward way to guide pod placement towards nodes that meet certain criteria, such as hardware capabilities, geographical location, or any other user-defined characteristic. - Allowed Topologies: Users can set specific rules for where share manager pods should go in the cluster by using `storageClass.allowedTopologies`. This setting is turned into affinity rules which are applied to a share manager pod. The affinity helps decide which nodes the pod can be placed on, based on the labels of those nodes. - Tolerations: Users can define tolerations for share manager pods within the storage class by setting storageClass.parameters["shareManagerTolerations"]. These tolerations allow share manager pods to be scheduled on nodes with matching taints. ### API changes `None` ## Design ### Implementation Overview When the share manager controller is in the process of reconciling a share manager, it attempts to search the associated storage class from the `persistentVolume.spec.StorageClassName`. If the storage class is nonexistent, node selectors and allowed topologies will also be absent, leading to a neglect of the share manager pod's locality. If the associated storage class is present, the system reads `storageClass.parameters["shareManagerNodeSelector"]` and `storageClass.allowedTopologies`. The node selectors specified in `storageClass.parameters["shareManagerNodeSelector"]` are combined with the global selectors from system-managed-components-node-selector and are applied to the share manager pod. Additionally, the system translates the `storageClass.allowedTopologies` into affinity rules, which are then applied to the configuration of the share manager pod as well. For tolerations, users are able to to specify these in the storage class through `storageClass.parameters["shareManagerTolerations"]`. These specified tolerations are combined with global tolerations defined under the global setting `taint-toleration`, enabling share manager pods to be allocated to nodes that have compatible taints. Once the share manager pod is allocated to a suitable node, the node's name is set to the volume attachment ticket. ### Test plan 1. Test RWX volumes from a storage class with `parameters["shareManagerNodeSelector"]`. 1. Test RWX volumes from a storage class with `storageClass.allowedTopologies`. 1. Test RWX volumes from a storage class with `storageClass.Parameters["shareManagerTolerations"]`. ================================================ FILE: enhancements/20240423-longhorn-commandline-interface.md ================================================ # Longhorn Commandline Interface (longhornctl) ## Summary Navigating Longhorn's troubleshooting and manual operations documents can be challenging. This proposal introduces the `longhornctl` Longhorn Command-Line Interface (CLI) to enhance user experience. This CLI steamlines operations, making Longhorn's manual operations simpler and more intuitive for users. ### Related Issues https://github.com/longhorn/longhorn/issues/7927 ## Motivation ### Challenges With Current Operations Currently, Longhorn users often face challenges when performing manual operations such as troubleshooting and non-custom-resource operations. These challenges include: - The need to refer to extensive documentation for each operations. - Complex and time-consuming steps for one-time operations. - Difficulty in gathering information for troubleshooting. - Manual processes for retrieving data and information during Longhorn failures. ### Goals This proposal's primary goal is to introduce a Longhorn CLI (`longhornctl`), laying the foundation for simplifying manual Longhorn operations. The specific objective include: - Providing a single, unified interface for common operations. - Reducing the time and effort required for one-time operations. - Streamlining troubleshooting by automating the information collection and error identification. - Simplifying data retrieval from Longhorn volumes during failures. #### Initial Implementation To demonstrate the capabilities of `longhornctl`, the initial Proof of Concept (PoC) will focus on the following operations: - Preflight installation/uninstallation - Preflight checking - Volume trimming - Replica exporting Additional operations will be developed on a case-by-case basis to further enhance the `longhornctl` CLI. ### Non-goals [optional] While this proposal aims to address fundamental manual operations, it does not encompass every existing possible troubleshooting scenario or operation. The focus is on establishing a structured CLI for common use cases. ## Proposal ### User Stories #### Story 1: Preparation for Longhorn As a Longhorn user, I want to prepare my environment using the `longhornctl` to easily set up my environment without the need for manual installations or searching through documents. - Before this enhancement: Users had to refer to documents to either prepare the environment using the Longhorn provided manifest or follow a detailed document to manually set up the environment for each cluster hosts. - After this enhancement: Users can simplify Longhorn prerequisites setup with a single command. #### Story 2: One-time operation As a Longhorn user, I want to perform one-time operations using the `longhornctl` to execute tasks without navigating the Longhorn UI or remembering complex commands. - Before this enhancement: User need to remember or lookup steps for one-time operations, a process that could be complex and time-consuming. - After this enhancement: One-time operations are consolidated into a uniflied CLI, removing the need to remember and following complex steps. This improves efficiency and reduce the likelihood of errors. #### Story 3: Troubleshoot As a Longhorn user, I want to troubleshoot Longhorn using the `longhornctl` to quickly identify and resolve issues. - Before this enhancement: Troubleshooting Longhorn requires manual information gathering from various sources, a time-consuming process. - After this enhancement: The troubleshooting process is streamlined, allowing user to quickly collect information and pinpoint issues. This enhancement improves efficiency and reduces the time needed to resolve Longhorn issues. #### Story 4: Data-retrieval As a Longhorn user, I want to retrieve data from Longhorn volumes using the `longhornctl` when Longhorn encounters failures. - Before this enhancement: User had to navigate through complex steps to manually mount Longhorn volumes for data access, a time-consuming and error-prone process. - After this enhancement: User an easily export data to a specified targeting path using the `longhornctl`, simplifying the process and enhancing the user experience during the Longhorn failures. ### User Experience In Detail When a user execute the `longhornctl`, they should encounter clear and informative output to enhance their understanding and usage. This includes: - Detailed command descriptions: Each command is accompanied by a detailed description, providing users with a comprehensive overview of its purpose and usage. - Help menu at each command layer: User can find help menu at each layer of the command and subcommands that allows user to get detailed information without navigating through external documentation. - Clear messages and log control: User have the ability to change the log level, allowing them to control the verbolity of output. This ensures that user receive clear logs and results, making it easier to understand the CLI's action and responses. Examples: - Showing help at root command: ```shell > ./bin/longhornctl --help Commands for managing Longhorn Install And Uninstall Commands: install Install Longhorn extensions uninstall Uninstall Longhorn extensions Operation Commands: trim Longhorn trim commands export Export Longhorn Troubleshoot Commands: check Check Longhorn get Get Longhorn resources Other Commands: global-options Print global options inherited by all scommands Use "longhornctl --help" for more information about a given command. ``` - Showing help at subcommand: ```shell > ./bin/longhornctl trim --help Longhorn trim commands Available Commands: volume Trim a Longhorn volume Use "longhornctl trim --help" for more information about a given command. > ./bin/longhornctl trim volume --help Trim a Longhorn volume Options: --longhorn-namespace='longhorn-system': Longhorn namespace --name='': Longhorn volume name Usage: longhornctl trim volume [flags] [options] Use "longhornctl trim options" for a list of global command-line options (applies to all commands). ``` - Clean notification for missing command option flag: ```shell ERROR[2024-04-23T13:25:30+08:00] Longhorn volume name (--name) is required ``` - Clear output of the result: ```shell INFO[2024-04-23T15:19:16+08:00] Successfully export replica: volumes: pvc-65789149-430e-41a9-944a-c1f3dc2dc5db: - replicas: - node: gke-lh-c3y1huang-6lv-lh-c3y1huang-6lv-b30251b0-tq57 exportedDirectory: /tmp/foo/pvc-65789149-430e-41a9-944a-c1f3dc2dc5db INFO[2024-04-23T15:19:16+08:00] Run 'longhornctl export replica stop' to stop exporting the replica ``` ``` INFO[2024-06-11T14:48:58+08:00] Retrieved preflight checker result: ip-10-0-2-5: info: - Service iscsid is running - NFS4 is supported - Package nfs-client is installed - Package open-iscsi is installed - CPU instruction set sse4_2 is supported - HugePages is enabled - Module nvme_tcp is loaded - Module uio_pci_generic is loaded ip-10-0-2-71: error: - Neither iscsid.service nor iscsid.socket is running info: - NFS4 is supported - Package nfs-client is installed - Package open-iscsi is installed - CPU instruction set sse4_2 is supported - HugePages is enabled - Module nvme_tcp is loaded - Module uio_pci_generic is loaded ip-10-0-2-248: info: - Service iscsid is running - NFS4 is supported - Package nfs-client is installed - Package open-iscsi is installed - CPU instruction set sse4_2 is supported - HugePages is enabled - Module nvme_tcp is loaded - Module uio_pci_generic is loaded ``` ### API changes Not applicable in this context. ## Design The `longhornctl` will be based on the existing https://github.com/longhorn/cli, originally designed for Longhorn preflight operations. ### Command #### Migration to `cobra` The command library will migrate from `urfave` to `cobra` for managing complex sub-command groupings and more powerful and customizable help menu. #### Command Groupings Introduce command groupings to enhance user understandings: ```shell > ./bin/longhornctl help Longhorn commandline interface for managing Longhorn Install And Uninstall Commands: install Install Longhorn extensions uninstall Uninstall Longhorn extensions Operation Commands: trim Longhorn trim commands export Export Longhorn Troubleshoot Commands: check Check Longhorn get Get Longhorn resources Other Commands: global-options Print global options inherited by all scommands Use "longhornctl --help" for more information about a given command. ``` #### Command Layer Conventions Commands will follow a structured format of `verb (primary action)` + `noun (item/entity)` + `specific action`. ```golang const ( // The first layer of subcommands (verb) SubCmdCheck = "check" SubCmdExport = "export" SubCmdGet = "get" SubCmdInstall = "install" SubCmdTrim = "trim" SubCmdUninstall = "uninstall" // The second layer of subcommands (noun) SubCmdPreflight = "preflight" SubCmdReplica = "replica" SubCmdVolume = "volume" // The third layer of subcommands (specific action to take concerning the action and item of the previous layers) SubCmdStop = "stop" ) ``` With these conventions, each command is structured logically: ``` Example usage: - longhornctl install preflight - longhornctl get replica - longhornctl export replica stop ``` #### Example Command Implementation ```golang func NewCmdGetReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { var replicaGetter = replica.Getter{} cmd := &cobra.Command{ Use: consts.SubCmdReplica, Short: "Get Longhorn replica information", Long: `This command retrieves detailed information about Longhorn replicas. The information can be used for troubleshooting and understand the state of your replicas. By default, the command retrieves information about all replicas in the system. You can optionally filter the results by providing: - Replica Name: Use the --name flag to specify the replica name you want the details for. - Volume Name: Use the --volume flag to filter replicas based on the volume they belong to.`, PreRun: func(cmd *cobra.Command, args []string) { replicaGetter.Image = globalOpts.Image replicaGetter.KubeConfigPath = globalOpts.KubeConfigPath }, Run: func(cmd *cobra.Command, args []string) { logrus.Infof("Initializing replica getter") if err := replicaGetter.Init(); err != nil { utils.CheckErr(errors.Wrapf(err, "Failed to initialize replica getter")) } logrus.Infof("Running replica getter") result, err := replicaGetter.Run() if err != nil { utils.CheckErr(errors.Wrapf(err, "Failed to run replica getter")) } logrus.Infof("Completed replica getter:\n %v", result) }, PostRun: func(cmd *cobra.Command, args []string) { logrus.Debugf("Cleaning up replica getter") if err := replicaGetter.Cleanup(); err != nil { utils.CheckErr(errors.Wrapf(err, "Failed to cleanup replica getter")) } }, } utils.SetGlobalOptionsRemote(cmd, globalOpts) cmd.Flags().StringVar(&replicaGetter.ReplicaName, consts.CmdOptName, "", "Specify the name of the replica to retrieve information (optional).") cmd.Flags().StringVar(&replicaGetter.VolumeName, consts.CmdOptLonghornVolumeName, "", "Specify the name of the volume to retrieve replica information (optional).") cmd.Flags().StringVar(&replicaGetter.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, "/var/lib/longhorn", "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the location of longhorn-disk.cfg (optional).") return cmd } ``` - Receiver instance (`replicaGetter`): this is used to handle operation related to the command, such as `Init()`, `Run()`, `Validate()`, `Cleanup()`, etc. - Short Description (`Short`): provide a brief description of the command, helping user understand its purpose at a glance. - Long Description (`Long`): a detailed explanation of the command, providing user with comprehensive information about its functionality. - PreRun Function: - Assigns global options to the `replicaGetter` instance - Validate the command options assigned to the `replicaGetter` instance. - Run Function: - Invoke receiver `Init()` for initializing `replicaGetter`. - Invoke receiver `Run()` to run the actual operation. - Check for errors of the invocation, exit with a non-zero code if an error occurs. - PostRun Function: - Invoke receiver `Cleanup()` to clean up resources used by the `replicaGetter` after command execution. - Global Options (`SetGlobalOptionsRemote`): - Sets global options for the subcommand, ensuring consistency and ease of use across all subcommands. - Command Flags: - Defines flag for the command, allowing users to provide additional options and assign it to the `replicaGetter` instance. #### Remote (longhornctl) The `longhornctl` command is responsible for managing the required resources within the Kubernetes cluster for in-cluster operations, such as ConfigMap, DaemonSet, etc. - Resource Management: This command handles the creation and deletion of the resources like ConfigMap and DaemonSet within the Kubernetes cluster. For instance, the DaemonSet that is responsible for running operations related to the command actions. - Monitoring: The `longhornctl` command monitors the status of the DaemonSet containers. This proactive monitoring block ensures that actions are completed and allows for error handling. - Error Handling and Log Retrieval: In case of an action failure, the command retrieves the logs of the specific container involved. These logs shall provide users with clear understandings of the failure. - Results Output: For actions that yields a collection of information, the command formats the presents the result in a user-friendly YAML formal. These ensures that users can easily interpret the output. #### Local (longhornctl-local) The `longhornctl-local` command is designed to run within a DaemonSet pod inside the Kubernetes cluster, focusing on managing interactions within the cluster environment, including both in-cluster and host operations. - Host interactions: When executed within the Daemonset pod, the `longhornctl-local` command can interact with the host system. This could involve tasks such as: - Managing storage devices. - Interacting with the host filesystem. - Executing system-level commands. ### Code Structure - cmd: Contains the main command files for both local and remote binaries. Keep the directories separate for clarity. - /local - /subcmd - longhornctl-local.go - /remote - /subcmd - longhornctl.go - /dapper: Houses the build script for creating local and remote binaries. - build: ```bash # Binary runs local to the Kubernetes cluster build_app local longhornctl-local # Binary runs remote to the Kubernetes cluster build_app remote longhornctl ``` - /pkg - /consts: Centralized constants. - /local: Package for local operations to the Kubernetes cluster. - /remote: Package for remote operations to the Kubernetes cluster. - /types: Centralized types definitions to ensure consistency. - /utils: Contains utilities categorized by purpose, specific for this repository (For common utilities, use `go-common-lib`). Example: ``` cli | +-- cmd | +-- local .......................... Command for operations within Kubernetes cluster. | | +-- subcmd ....................... Subcommands specific to local operations. | | | +-- check.go | | | +-- get.go | | | +-- install.go | | | +-- trim.go | | +-- README.md | | +-- longhornctl-local.go ......... Main command file for local operations. | +-- remote ......................... Commands for operations outside the Kubernetes cluster. | +-- subcmd ....................... Subcommands specific to remote operations. | | +-- check.go | | +-- get.go | | +-- install.go | | +-- trim.go | +-- longhornctl.go ............... Main command file for remote operations. +-- dapper | +-- build .......................... Build file to create local/remote binaries. +-- pkg +-- consts ......................... Constants used throughout the project. | +-- cmd.go | +-- env.go | +-- longhorn.go | +-- prflight.go | +-- replica.go | +-- spdk.go | +-- longhornctl.go | +-- volume.go +-- local .......................... Package for local command operations. | +-- preflight | | +-- checker.go ................. Logic for checking pre-flight conditions. | | +-- installer.go ............... Logic for installing packages. | | +-- pkgmgr ..................... Package managers for various systems. | | +-- apt.go | | +-- pacman.go | +-- replica | | +-- getter.go .................. Logic for getting replica information. | | +-- exporter.go ................ Logic for exporting replica. | +-- volume | | +-- trimmer.go ................. Logic for trimming volume. +-- remote ......................... Package for remote command operations. | +-- preflight | | +-- checker.go ................. Logic for Kubernetes resource handling. | +-- replica | | +-- getter.go .................. Logic for Kubernetes resource handling. | +-- volume | +-- trimmer.go ................. Logic for Kubernetes resource handling. +-- types | +-- cmd.go ....................... Definitions for command-related types. | +-- pod.go ....................... Definitions for Kubernetes Pod related types. | +-- replica.go ................... Definitions for Longhorn Replica related types. | +-- volume.go .................... Definitions for Longhorn Volume related types. +-- utils +-- kubernetes | +-- runtime.go ................. Utility functions related to Kubernetes runtime. +-- longhorn | +-- longhorn.go ................ Utility functions related to Longhorn. +-- cmd.go ....................... Utility functions related to commands. +-- utils.go ..................... Utility general functions. ``` ### Feature #### Install Preflight ##### Remote Command (`longhornctl install preflight`) - Introduce command to prepare and create a `longhorn-preflight-installer` DaemonSet. - Include `--operating-system` command option to accommodate operating systems that do not include a package manager, such as container-optimized OS (COS). - When no operating system is specified, create a DaemonSet include: - An init-container executing `longhornctl-local install preflight`. - A container running a pause image as an indicator to signal the completion of the `longhornctl-local` command in the init-container. - When operating system is specified to `cos`, create: - a ConfigMap include: - The entrypoint.sh script. - a DaemonSet include: - Container running the `entrypoint.sh` script. Ref: [longhorn-gke-cos-node-agent.yaml ](https://github.com/longhorn/longhorn/blob/2cbedf3902e72e347eed1eb8a45768462a4dd76b/deploy/prerequisite/longhorn-gke-cos-node-agent.yaml) - Readiness probe for successful installation indication. - Liveness probe for error detection. ##### Local Command (`longhornctl-local install preflight`) - Transition commandline library from `urfave` to `cobra`. - Refactor command returns for error handling. - Retain the existing functionality for installation via package manager. #### Check Preflight ##### Remote (`longhornctl check preflight`) - Introduce command to prepare and create a `longhorn-preflight-checker` DaemonSet. - The DaemonSet include: - An init-container executing `longhornctl-local check preflight`. - A container running a pause image as an indicator to signal the completion of the `longhornctl-local` command in the init-container. - Post-command cleanup: - Upon command completion, cleanup the DaemonSet. ##### Local (`longhornctl-local check preflight`) - Transition commandline library from `urfave` to `cobra`. - Refactor command returns for error handling. - Retain the existing functionality for checking via package manager and services. #### Trim Volume ##### Remote (`longhornctl trim volume`) - Introduce command to prepare and create a `longhorn-volume-trimmer` DaemonSet. - The DaemonSet include: - An init-container executing `longhornctl-local trim volume`. - A container running a pause image as an indicator to signal the completion of the `longhornctl-local` command in the init-container. - Post-command cleanup: - Upon command completion, cleanup the DaemonSet. ##### Local (`longhornctl-local trim volume`) - Implement logic to detect and execute `fstrim` for `RWO` and `RWX` volumes. #### Get Replica ##### Remote (`longhornctl get replica`) - Introduce command to prepare and create a `longhorn-replica-getter` DaemonSet. - The DaemonSet include: - A shared volume mount for the result output to `replica.json`. - An init-container executing `longhornctl-local get replica`. - A container running `cat "replica.json"`. - When the DaemonSet is running successfully, retrieve the log of the container outputting the `replica.json`, then unmarshal JSON to YAML and output to the users. - Post-command cleanup: - Upon command completion, cleanup the DaemonSet. ##### Local (`longhornctl-local get replica`) - Implement logic to collect replica information from host directories and `volume.meta` files: ```golang type ReplicaCollection struct { Replicas map[string][]*ReplicaInfo `json:"replicas" yaml:"replicas"` } // ReplicaInfo holds information about a replica. type ReplicaInfo struct { Node string `json:"node,omitempty" yaml:"node,omitempty"` Directory string `json:"directory,omitempty" yaml:"directory,omitempty"` IsInUse *bool `json:"isInUse,omitempty" yaml:"isInUse,omitempty"` VolumeName string `json:"volumeName,omitempty" yaml:"volumeName,omitempty"` Metadata *lhmgrutil.VolumeMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` Error string `json:"error,omitempty" yaml:"error,omitempty"` ExportedDirectory string `json:"exportedDirectory,omitempty" yaml:"exportedDirectory,omitempty"` } ``` - Output the collected information to `replica.json` within the shared volume mount. #### Export Replica ##### Remote (`longhornctl export replica`) - Introduce command to prepare and create: - A `longhorn-replica-exporter` ConfigMap - A `longhorn-replica-exporter` DaemonSet - The ConfigMap includes and `entrypoint.sh` responsible for: - Read the replica information from the `replica.json` in the shared volume mount. - Pausing the daemonset pod if the replica is not on the pod node. - Checking if the replica is in use. - Running `launch-simple-longhorn ${VOLUME_NAME} ${VOLUME_SIZE} &` - Creating pre-stop script. - Mouting the device to the target host path. - The DaemonSet includes: - A shared volume mount for the result output to `replica.json`. - An init-container executing `longhornctl-local get replica` for collecting the replica information and output to the `replica.json` in the shared volume mount. - A container running `entrypoint.sh` in the engine image. - A readiness probe to check for action completion. - A pre-stop hook to execute pre-stop script for the host mount point cleanup. ##### Local (`None`) No local command for replica exporting, as the dependent `launch-simple-longhorn` is built in the engine image. ### Test plan 1. Command execution testings: - Install and uninstall commands - Operating commands - Troubleshooting commands - Global options printing 1. Command help testings: - Run `longhornctl --help` and verify that all sub-commands are displayed - Run `longhornctl --help` for each sub-command and verify detailed help information is provided. 1. Error Handling Testing: - Execute commands with incorrect or missing options, and verify error message are displayed to guide users on correct usage. 1. Functionality Testing: - Test each command with valid input to ensure they perform the intended operations. - Verify correct behavior and output for each command. 1. Cross-platform testing: - Test `longhornctl` on different operating systems to ensure compatibility. ### Upgrade strategy No specific upgrade strategy is needed currently for this phase of the project. ## Note [optional] `None` ================================================ FILE: enhancements/20240426-backing-image-enhancement.md ================================================ # Backing Image Enhancement ## Summary This feature enhances the management of BackingImages in Longhorn. With **High Availability**, Longhorn maintains more than one BackingImage copy in the cluster to avoid data loss. Additionally, by incorporating **nodeSelector and diskSelector**, Longhorn can store the BackingImages on specific nodes and disks to improve space efficiency. Lastly, with the addition of **eviction handling**, Longhorn can move the BackingImage to other nodes when users request eviction on the node. ### Related Issues - https://github.com/longhorn/longhorn/issues/2856 - https://github.com/longhorn/longhorn/issues/6526 ## Motivation ### Goals #### HA 1. A user can set global high available factor for all the BackingImages or specify the number for each BackingImage. 2. Longhorn will maintain the number of BackingImage copy in the cluster. 3. Longhorn will delete the extra BackingImage copy if it is not used for a while and the number of copy is more than the factor in the cluster. #### nodeSelector and diskSelector 1. A user can add `nodeSelector` and `diskSelector` when creating BackingImage 2. The BackingImage copies will be located on the nodes and disks with the corresponding `tags`. 3. When the node or disk is disabled for scheduling, the BackingImage copies can't be placed on the node or disk. 4. Replica is not able to be scheduled on the nodes and disks where Backingimage can not be stored. #### Eviction Handling 1. When a user disables the scheduling and set eviction request on the node or disk, Longhorn will move the BackingImage copies to other nodes or disks. ### Non-goals #### Eviction Handling When Cordoning or Draining Node. We only support evicting BackingImage copy when manually set eviction request to the node or disk. That is, if a user want to cordon or drain the node. Longhorn won't evict the BackingImage copies automatically like what it does for the Replicas. In this case, the user needs to set eviction request to the node or disk first before draining the node. The reason is that we will need to set PDB to protect BackingImageManager from deleting during draining process. It increases the complexity of the process and adding risk of stucking in the process. ## Proposal ### User Stories #### HA Before this feature, users might lose the BackingImage copy if the node that holds the only copy in the cluster crashes or gets drained. Users then need to prepare the BackingImage again. With this feature, Longhorn maintains the number of copy in the cluster to reduce the possibility of the data loss. #### Node Selector and Disk Selector Before this feature, Longhorn stores the BackingImage copy on a random node and a random disk. When a replica needs the BackingImage, Longhorn needs to copy the BackingImage to the disk where replica is. Similar to Volume Replicas scheduling, with adding `nodeSelector` and `diskSelector` to the BackingImage, Longhorn will place the BackingImage copies to the specific nodes or disks. BackingImages will be stored in the same set of nodes and disks as Replicas if they have the same `nodeSelector` and `diskSelector`. It improves space efficiency. #### Replica Scheduling Since replica won't be able to start on the node where BackingImage can not be stored. Thus, we will not schedule the replica to the node and disk where the BackingImage can not be stored because of the `nodeSelector` and `diskSelector`. #### Eviction Handling Before this feature, Longhorn won't move the BackingImage copies to other nodes when the node is set to eviction requested. With this feature, a user can manually request eviction on the node or disk to evict all the BackingImage copies to other nodes and disks. ### API changes ## Design - HA ### Implementation Overview #### CRD - Add `numOfCopies` to BackingImage #### BackingImage Controller - After first BackingImage is ready, the controller starts maintaining the number of copies in the clusters. - If the number is lower, then the controller picks another valid node and disk for the BackingImage copy. Increase one every time until it is equal or larger than the number. - If the copy is not used for a while and the number of copy is larger than the setting, Longhorn deletes the unused BackingImage copies. (already implemented before: [ref](https://github.com/longhorn/longhorn-manager/blob/v1.6.1/controller/node_controller.go#L1152)) ## Design - NodeSelector and DiskSelector ### Implementation Overview #### CRD - Add `diskSelector` and `nodeSelector` to the BackingImage #### BackingImage Controller - When selecting node/disk for the BackingImage, Longhorn needs to follow the `diskSelector` and `nodeSelector` settings. #### Replica Scheduler - For the node candidates and the disk candidates, we also check if they are available for the BackingImage used by the Replica. If the BackingImage can not be stored on the node or the disk, we will not schedule the Replica to the node either. ## Design - Eviction Handling ### Implementation Overview #### CRD - Change the BackingImage Spec to add `EvictionRequested` for each BackingImage copy on disk. ``` type BackingImageSpec struct { Disks map[string]string `json:"disks"` Checksum string `json:"checksum"` SourceType BackingImageDataSourceType `json:"sourceType"` SourceParameters map[string]string `json:"sourceParameters"` } ``` ``` type BackingImageSpec struct { Disks map[string]*BackingImageDiskFileSpec `json:"disks"` Checksum string `json:"checksum"` SourceType BackingImageDataSourceType `json:"sourceType"` SourceParameters map[string]string `json:"sourceParameters"` } type BackingImageDiskFileSpec struct { EvictionRequested bool `json:"evictionRequested"` } ``` ##### Node Controller - When the node or disk is set to `evictionRequested = true`, node controller will update EvictionRequested for all the BackingImage copy on the node. #### BackingImage - `replenishBackingImageCopies()` - if `nonFailedCopies >= MinNumberOfCopies`, we check if we need to replenish one copy for eviction - replenish one if `NonEvictingCount < MinNumberOfCopies` - if `nonFailedCopies < MinNumberOfCopies` - replenish one copy to meet the MinNumberOfCopies requirement. - `cleanupEvictionRequestedBackingImageCopies()` - If there is no non evicted healthy copy, don't delete the copy. - Otherwise, delete the evicted copy. --- ### Test plan 1. HA - Create a BackingImage with `minNumberOfCopies = 2` - After creation, it will sync the file to another node/disk immediately - Update the `Backing Image Cleanup Wait Interval to 1 min` - Update the `minNumberOfCopies = 1` - The extra Backing Image will be cleaned up 2. nodeSelector/diskSelector - Set node1 with `nodeTag: [node1], diskTag:[disk1]` - Create a BackingImage with - `minNumberOfCopies = 2` - `nodeSelector = [node1]` - `diskSelector = [disk1]` - After creation, the first BackingImage copy will be on node1, disk1 - But the second one will never show up - The log will show `unable to get a ready node disk` 3. Different nodeSelector and diskSelector as Replicas (Negative Test) - Set `nodeTag: [node1], diskTag: [disk1]` to node1/disk1 - Set `nodeTag: [node2], diskTag: [disk2]` to node2/disk2 - Create a BackingImage with following specs - `minNumberOfCopies = 1` - `nodeSelector = [node1]` - `diskSelector = [disk1]` - Create a Volume with following specs and attach to node2 - `numberOfReplicas = 1` - `nodeSelector = [node2]` - `diskSelector = [disk2]` - The volume condition `Scheduled` will be `false` because the replica is not able to be scheduled. 3. Eviction - 1 - Create BackingImage with one copy - Evict the node where copy is on - The BackingImage will first create another copy in another node - The evicted copy will be deleted 4. Eviction - 2 - Set BackingImage `minNumberOfCopies=1` - Create BackingImage with two copies - Evict the node where one of the copy is on - The evicted copy will be deleted 5. Eviction - 3 (Negative Test) - Set nodeTag: [node1], diskTag:[disk1] - Create a BackingImage with following settings to place the copy on node1 - `minNumberOfCopies = 1` - `nodeSelector = [node1]` - `diskSelector = [disk1]` - Evict node1 - The copy would not be deleted because it is the only copy - The copy can't be duplicated to other nodes because of the selector settings. ### Upgrade strategy None ## Note [optional] Additional notes. ================================================ FILE: enhancements/20240516-pre-pull-images.md ================================================ # Pre-Pull Images ## Summary We will pre-pull the share-manager image on every worker node to speed up the startup of share-managers. ### Related Issues https://github.com/longhorn/longhorn/issues/8376 ## Motivation ### Goals - Pre-pull the share-manager image on all worker nodes. ### Non-goals [optional] `None` ## Proposal - Sidecar containers in the `longhorn-manager` DaemonSet to pre-pull images ### User Stories With the pre-pull images mechanism, users can deploy RWX volume and start to use volumes faster. It's also useful for [Share manager HA mechanism](https://github.com/longhorn/longhorn/issues/6205). There is the [garbage collection of unused containers and images feature](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#containers-images) of kubernetes. The `kubelet` performs garbage collection on unused images every 2~5 minutes and on unused containers every minute so we have sidecar containers in the `longhorn-manager` DaemonSet to prevent the pre-pull images from being deleted by the garbage collection of unused containers and images. ### API changes `None` ## Design ### Implementation Overview - Add containers in the `longhorn-manager` DaemonSet: ```yaml spec: containers: - name: longhorn-manager ... - name: pre-pull-share-manager-image imagePullPolicy: IfNotPresent image: longhornio/longhorn-share-manager:master-head command: ["sh", "-c", "echo longhorn-share-manager image pulled && sleep infinity"] ``` ### Test plan - Fresh install/Upgrade: - After install or upgrade, the share-manager image are pulled on the each worker node. ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20240522-storage-network-for-rwx-volumes.md ================================================ # Storage Network For Read-Write-Many (RWX) Volume ## Summary This proposal outlines extending Longhorn's storage network feature to support Read-Write-Many (RWX) volumes. Having graduated from experimental status, RWX volumes should now be able to use storage network for data traffic, similar to Read-Write-Once (RWO) volumes. The enhancement enables data network isolation and potentially improve performance in specific network configuration that utilize storage network. ### Related Issues https://github.com/longhorn/longhorn/issues/8184 ## Motivation ### Goals Enable users to utilize a dedicated network interface for RWX volume data traffic with Multus, similar to Read-Write-Once (RWO) volumes. This provides data network isolation and possibly can enhance performance in specific configurations. ### Non-goals [optional] - This proposal does not involve performance evaluation of RWX volumes between storage networks and cluster networks. ## Proposal ### User Stories #### Story 1: RWX Volume Data Network Segregation As a Longhorn user, I want to use a dedicated network interface for RWX volume data traffic. - Before: Data network segregation was unavailable for RWX volumes. The data network segregation only works in the RWO volumes, and the data traffic between the mount point (client) and the share manager pod still goes through the cluster network. - After: User can use the storage network for RWX volumes, enabling network segregation. #### Story 2: Pre-existing and New RWX Volume Workload As a Longhorn user, I have storage network configured, and existing RWX volume workload before the v1.7.0 upgrade. - Existing workload will continue using the cluster network without disruption. - New RWX volumes also user the cluster network. - User can enable the `storage-network-for-rwx-volume-enabled` setting to apply the storage network to re-attached RWX volumes. - Once the storage network is applied to the RWX volumes, the workload pod need to be restart when the CSI plugin pod restarts. User can automate this by enabling the `auto-delete-pod-when-volume-detached-unexpectedly` setting. ### Story 3: Using Storage Network For RWO Volume Workload Only As a Longhorn user, I want to use the storage network for only RWO volumes to avoid restarting my RWO volume workload pods when the CSI plugin pod restarts. - User can control the application of the storage network to RWX volues by using the `storage-network-for-rwx-volume-enabled` setting. ### User Experience In Detail Users will be able to create RWX volumes that uses the storage network for data traffic. Key user-facing changes include: - **CSI Annotations:** Share manager pods and CSI plugin pods will be annotated with the storage network for NFS client mounting when the `storage-network-for-rwx-volume-enabled` setting is enabled. This annotation will only be applied once all RWX volumes have been detached. - **Headless Service:** storage network enabled RWX volume will use a headless service and a custom endpoint, removing dependency on ClusterIP, which is not suitable for Multus networking. - **Workload Pod Restart:** When a CSI plugin pod restarts, the NFS share mountpoint becomes unavailable, rendering any attempt to access it unresponsive. User can manually restart the workload pod, or they can enable the `auto-delete-pod-when-volume-detached-unexpectedly` setting. This allows Longhorn to delete the associated workload pod when it is using the storage-network enabled RWX volume. If the pod is managed by a controller, the kubelet will create a new pod. This allows the CSI controller to handle volume remounting through the CSI node server. - **Upgrade to Longhorn v1.7.0:** Existing RWX volumes remain unchanged if the storage network is configured before the upgrade. To apply the storage network for RWX volumes, detach all RWX volumes, enable the `storage-network-for-rwx-volume-enabled` setting, and reattach the volumes. ### API changes No API changes are required for this proposal. ## Design ### Storage Network For RWX Volume Enabled Setting. A new setting, `storage-network-for-rwx-volume-enabled`, allows controlling the application of the storage network for RWX volumes. When the CSI plugin pod of a workload using the storage network enabled RWX volume restarts, the workload pod also needs a restart to re-establish the NFS client mount connection. This setting enabled user to manage whether the storage network should be applied to the RWX volumes. For RWX volumes using the cluster network, no restart is required during the CSI plugin pod restarts. ```golang SettingDefinitionStorageNetworkForRWXVolumeEnabled = SettingDefinition{ DisplayName: "Storage Network for RWX Volume Enabled", Description: "This setting allows Longhorn to use the storage network for in-cluster data traffic for RWX (Read-Write-Many) volume. \n\n" + "To apply this setting to existing RWX volumes, they need to be reattached. \n\n" + "WARNING: \n\n" + " - Enabling this setting will allow the CSI plugin pod to restart with the storage network annotatation if all RWX volumes are detached. \n\n" + " - The RWX volumes will be mounted with the storage network within the CSI plugin pod container network namespace. \n\n" + " - Consequently, restarting the CSI plugin pod may lead to unresponsive RWX volume mounts. If this occurs, you will need to restart the workload pod to re-establish the mount connection. \n\n" + " - Alternatively, you can enable the 'Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly' setting. \n\n", Category: SettingCategoryDangerZone, Type: SettingTypeBool, Required: false, ReadOnly: false, Default: "false", } ``` ### CNI Annotation Introduce CSI annotation for RWX volume pod stack to enable NFS client mounting over the storage network: - Share manager pod - CSI plugin pod The setting controller will apply these annotations once `storage-network-for-rwx-volume-enabled` setting is enabled and all RWX volumes are detached. ### Service Currently, Longhorn creates a share manager service auto-assigned with a cluster IP by the selector. The cluster IP is used for the share manager endpoint. However, the service selector is unaware of the Multus networking when the storage network is configured. Hence, for the RWX volumes, Longhorn will create a headless service and custom endpoint for the RWX volume resource stack. ```golang // Service service.Spec.ClusterIP = "None" ``` ```golang // Endpoint newObj := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: sm.Name, Namespace: sm.Namespace, OwnerReferences: datastore.GetOwnerReferencesForShareManager(sm, false), Labels: labels, }, Subsets: []corev1.EndpointSubset{}, } ``` The share manager of the storage network enabled RWX volume will use the service FQDN for its endpoint instead of the value from service `ClusterIP` field. ```yaml endpoint: nfs://pvc-117d3553-1a2c-4b42-aa49-2b221dfb5cd9.longhorn-system.svc.cluster.local/pvc-117d3553-1a2c-4b42-aa49-2b221dfb5cd9 ``` ### Kubernetes Endpoint Controller Introduce a new Kubernetes Endpoint controller to watch over share manager pod resources during *Add* and *Update* operations to sync the share manager details with the endpoint subset. ```golang desireSubset := corev1.EndpointSubset{ Addresses: []corev1.EndpointAddress{ { IP: storageIP, NodeName: &pod.Spec.NodeName, TargetRef: &corev1.ObjectReference{ Kind: types.KubernetesKindPod, Name: pod.Name, Namespace: pod.Namespace, UID: pod.UID, }, }, }, Ports: []corev1.EndpointPort{ { Name: "nfs", Port: 2049, Protocol: corev1.ProtocolTCP, }, }, } ``` ### Share Manager Process Namespaces Currently, the CSI plugin pod uses `HostPID` and mounts the host `proc` directory to be used by the custom mounter. This custom mounter wraps the commands (e.g., mount and umount) with `nsenter` to access the host namespaces, ensuring that the Kernel NFS client is tied to the host network namespace. This design ensures the NFS share mount persists after the CSI plugin pod restarts. A difference approach is required when the storage network is configured. Multus networks exist only in the Kubernetes network space. Therefore, for RWX volumes with a storage-network, the CSI plugin pod cannot run in the host network. When a RWX volume uses the storage network, the CSI plugin operate in its own network namespace. Since the NFS mount is managed by the Kernel module, the client is unaware of the client namespace is tied to the CSI plugin pod, resulting in a dangling mount after the CSI plugin pod restart. To workaround this, Longhorn will delete the workload pod (triggering its controller to restart the pod) to allow the CSI controller to handle the remount through the CSI node server's *NodeUnpublishVolume* and *NodePublishVolume*. ### Kubernetes Pod Controller Introduce a new responsibility to the Kubernetes pod controller: deleting workload pods that use the storage network enabled RWX volume when the CSI plugin pod restarts (during *Delete* operations). When a workload pod is deleted, its owning controller should create a new pod, initiating the normal volume attachment process. This process involves the CSI controller requesting the node server to unmount and mount the mount points, re-establishing the connection of the dangling mount. However, this approach results in a temporary disruption to the workload pod. Therefore, users can control this behavior using the `auto-delete-pod-when-volume-detached-unexpectedly` setting. The controller will not trigger RWX volume workload pod deletion in the following scenarios: - Storage network setting is not configured. - Storage network for RWX volume setting is disabled. ### Test plan 1. **Feature Testing:** Verify that the storage network supports RWX volumes and ensure the functionality of new settings. 1. **Regression Testing:** Cover both cluster network and storage network scenarios to ensure all functionalities work as expected. 1. **Resilience Testing:** Verify data accessibility after CSI plugin pod restarts for both cluster network and storage network scenarios. 1. **Cross-platform Testing:** Ensure RWX volume functionality on non-traditional operating systems such as Talos Linux and Container-Optimized OS for compatibility. 1. **Upgrade Testing:** Validate that existing RWX volumes function correctly after an upgrade. ### Upgrade strategy - **Existing RWX Volumes:** RWX volumes will not use the storage network unless the `storage-network-for-rwx-volume-enabled` setting is enable. Therefore, upgrading to Longhorn v1.7.0 will not affect existing RWX volume workloads even if the storage network was already configured before the upgrade. To enable the storage network for both pre-upgrade-existing and new RWX volumes after the upgrade, user needs to: 1. Detach all RWX volumes. 1. Enable the `storage-network-for-rwx-volume-enabled` setting. 1. Reattach the RWX volumes. - **Future Version Upgrades (v1.7.1, v1.7.2, etc.):** Given the storage network is enabled for RWX volumes, upgrading to a future version will cause the Longhorn NFS data mounts in associated workload pods to become unresponsive after the CSI plugin pod restarts. To resolve this issue, user have two options: 1. Enable the `auto-delete-pod-when-volume-detached-unexpectedly` setting before the upgrade. This allows Longohrn automatically restart the workload pod after the CSI plugin pod restart, re-establish the NFS client mount connection. 1. Manually restart the workload pods after the upgrade. > **Note:** Both options resolves the unresponsiveness issue, however, it will cause a brief interruption to the workload pod. ## Note [optional] - The [original issue](https://github.com/longhorn/longhorn/issues/8184) includes discussions about the workload data directory becoming unresponsive after the CSI plugin pod restarts. To address this, there is an initial consensus on a solution that involves deleting the workload pod. This approach effectively re-establish the mount point, ensuring continued data accessibility. - The [[BUG] Networking between longhorn-csi-plugin and longhorn-manager is broken after upgrading Longhorn to 1.7.0-rc3](https://github.com/longhorn/longhorn/issues/9223) includes reason for retaining the custom mounter and the use of *ClusterIP* for the regular RWX volume share manager endpoint. ================================================ FILE: enhancements/20240612-backing-image-encryption-and-clone-support.md ================================================ # BackingImage Encryption Support ## Summary Longhorn supports encrypted volumes by utilizing a Linux kernel-based disk encryption solution (LUKS, Linux Unified Key Setup). If users want to use BackingImage with encrypted Volume, the BackingImage needs to be encrypted as well. This feature allows users to clone the existing BackingImage and encrypt the content during cloning. ### Related Issues - https://github.com/longhorn/longhorn/issues/7051 ## Motivation ### Goals - Users can create a Encrypted Volume with the Encrypted BackingImage - Users can clone a BackingImage - Users can encrypt a BackingImage during the cloning process - Users can decrypt an encrypted BackingImage during the cloning process - For qcow2 image, if we want to encrypt it - We will temporarily create raw image from the qcow2 first and then encrypt it. - The result will be an encrypted raw image ### Non-goals [optional] - Users can change the encryption key of the encrypted BackingImage - Encrypt qcow2 image to a encrypted qcow2 image ## Proposal ### User Stories #### Utilize encrypted Volume with BackingImage Before this feature, users can not utilize an encrypted volume with an unencrypted BackingImage. Users will get `unsupported disk encryption format unknown data, probably partitions` error during mounting the device. The content inside the BackingImage needs to be encrypted using the same key as the Volume before mounting to the workload. Thus, we need to support users to encrypt the BackingImage in Longhorn. #### Clone the BackingImage With this feature, users can create an identical BackingImage by cloning another BackingImage. #### Encryption and Decryption With introducing cloning mechanism, we can encrypt and decrypt the content inside the BackingImage during the cloning process. Noted that, it means it will cost additional space if users want to encrypt or decrypte the BackingImage. The encrypted or decrypted BackingImage will be a new BackingImager in the system. ## Design ### Implementation Overview #### CRD Add a field to the BackingImage - Secret/SecretNamespace: - Pointing to the k8s secret containing the key used for encryption and decryption - We use `Secret` to check if the BackingImage is encrypted. If the BackingImage has `Secret` then it means it is encrypted. ``` type BackingImageSpec struct { Disks map[string]string `json:"disks"` Checksum string `json:"checksum"` SourceType BackingImageDataSourceType `json:"sourceType"` SourceParameters map[string]string `json:"sourceParameters"` Secret string `json:"secret"` SecretNamespace string `json:"secretNamespace"` } ``` #### Backing Image Data Source - Add a new source type: `clone` - parameters: - `source`: the source BackingImage - `encryption`: `encrypt`, `decrypt` and `ignore` - `secret`: the key used in the encryption and decryption. - Encryption - `encrypt`: Longhorn copy the BackingImage and apply the encryption - `decrypt`: Longhorn copy the BackingImage and decrypt the content - `ignoe`: Longhorn simply copy the BackingImage - If we are encrypting the BackingImage, copy the `Secret` and `SecretNamespace` from parameters to `BackingImage.Spec.Secret` and `BackingImage.Spec.SecretNamespace` - When choosing the node for the data source pod, we always choose one of the node and disk from the source BackingImage #### Webhook - If Source BackingImage has `Secret`, we don't allow to perform `"encrypt"`. (Forbid encrypted -> encrypted) - If Source BackingImage doesn't have `Secret`, we don't allow to perform `"decrypt"`. (Forbid unencrypted -> decrypted) - If the operation is to `"encrypt"` or `"decrypt"`, the `Secret` and `SecretNamespace` are needed. - User cannot change the `Secret` and `SecretNamespace`. #### Backing Image Manager - Data Source - When init the service, if the type is clone, then clone from source BackingImage by requesting sync service in the same pod. ```golang requestURL := fmt.Sprintf("http://%s/v1/files", client.Remote) // credential contains the crypto secret req, err := http.NewRequest("POST", requestURL, bytes.NewReader(encodedCredential)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") q := req.URL.Query() q.Add("action", "cloneFromBackingImage") // the source backing image q.Add("backing-image", sourceBackingImage) // we need uuid to get the path q.Add("backing-image-uuid", sourceBackingImageUUID) // can be ignore, encrypt or decrypt q.Add("encryption", encryption) q.Add("file-path", filePath) q.Add("uuid", uuid) q.Add("disk-uuid", diskUUID) q.Add("expected-checksum", expectedChecksum) ``` - When doing the cloning - The source file is the source BackingImage file on the same disk - If the operation is `"encrypt"` and the source file is `qcow2` - We create a temp raw image from the qcow2 image and use it as source file - Truncate the target file with the source file size. - If the operation is `"encrypt"`, we `+16MB` for LUKS meta size - If the operation is `"decrypt"`, we `-16MB` - If the operation is `"encrypt"` - Setup the loop device with the target file - `cryptsetup luksFormat` and `luksOpen` the device - set the target to the LUKS device - If the operation is `"decrypt"` - Setup the loop device with the source file - `cryptsetup luksOpen` the device - set the source to the LUKS device - copy all the data from source to the target - Need to write zeros since zeros also need to be encrypted. - For other cases we can skip zero (ref to the below operation table) #### LUKS related information - LUKS2 metadata size is 16MB - release: v2.1.0-ReleaseNotes - reference: https://gitlab.com/cryptsetup/cryptsetup/-/blob/master/docs/v2.1.0-ReleaseNotes#L27 #### operation table | - | unencrypted img | encrypted img | | -------- | -------- | -------- | | encrypt |allow (write zero)| x | | decrypt | x | allow (skip zero) | | ignore | allow (skip zero) | allow (skip zero) | #### UI - When Create Backing Image, when the `Created From` is `clone`, add following fields - `Backing Image`: the source BackingImage. - `Encryption`: 3 options for selection, `ignore`, `encrypt` and `decrypt`. - `Secret`: the secret name for the encryption or decryption. - `Secret Namespace`: the namespace of the secret. - If the BackingImage has `Secret` and `Secret Namespace` is not empty, show a "lock" icon besides it like encrypted volume. ### Test plan #### Raw Image Encryption - Create BackingImage - `https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.raw` - Create Secret and StorageClass ``` apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Your encryption passphrase" CRYPTO_KEY_PROVIDER: "secret" CRYPTO_KEY_CIPHER: "aes-xts-plain64" CRYPTO_KEY_HASH: "sha256" CRYPTO_KEY_SIZE: "256" CRYPTO_PBKDF: "argon2i" --- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-global provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "2" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" backingImage: "parrot-cloned-encrypted" backingImageDataSourceType: "clone" csi.storage.k8s.io/provisioner-secret-name: "longhorn-crypto" csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-stage-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" ``` - Create PVC ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-backing-image-pvc spec: accessModes: - ReadWriteOnce storageClassName: longhorn-crypto-global resources: requests: storage: 1Gi ``` - Create Pod to use the encrypted Volume with the encrypted BackingImage ``` apiVersion: v1 kind: Pod metadata: name: longhorn-simple-pod namespace: default spec: nodeSelector: kubernetes.io/hostname: kworker1 restartPolicy: Always containers: - name: ubuntu image: ubuntu:22.04 imagePullPolicy: IfNotPresent command: [ "/bin/bash", "-c", "--" ] args: [ "while true; do sleep 30; done;" ] volumeMounts: - name: vol mountPath: /data ports: - containerPort: 80 volumes: - name: vol persistentVolumeClaim: claimName: longhorn-backing-image-pvc ``` - Decrypt the Encrypted BackingImage ``` apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot-cloned-decrypt namespace: longhorn-system spec: sourceType: clone sourceParameters: backing-image: parrot-cloned-encrypted encryption: decrypt secret: longhorn-crypto secret-namespace: longhorn-system ``` - The checksum of the `parrot-cloned-decrypt` should be the same as `parrot` #### Qcow2 Image Encryption - Follow the same process, but change the BackingImage to qcow2 - `https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.qcow2` ### Upgrade strategy No Need ## Note [optional] Additional notes. ================================================ FILE: enhancements/20240801-delete-backup-in-the-backupstore-asynchronously.md ================================================ # Delete backup in the backupstore asynchronously ## Summary When deleting the backup, Longhorn deletes the backup in the backupstore using binary command synchrounosly. During the backup deletion, Longhorn creates a deletion lock in the backupstore to prevent other backups' creation. Backup creation fails immediately without retrying when encountering the deletion lock. This confuses and annoys users. This feature refactors the code to execute the backup deletion command asynchronously and introduces a new `Deleting` state to notify other backups to wait until the deletion process is complete. This feature aims to prevent creation failures caused by deletion lock issues. ### Related Issues - https://github.com/longhorn/longhorn/issues/8742 ## Motivation ### Goals - Delete the backup in the backupstore asynchronously. - When the Backup is being deleted, the state would be `Deleting` - Creation of other backups should be delayed until the backup deletion is complete. ## Design ### Implementation Overview #### Normal case To make deletion asynchronously, we can simply run the command in another go routine. Then we periodically check if the backup still exists in the backupstore before removing the finalizer. However, since the command runs in another go routine, we need to handle the scenario of command failure. Thus, we introduce an in-memory map as share memory for the go routine to notify the controller. ### Error handling ``` type DeletingStatus struct { State longhorn.BackupState ErrorMessage string } type BackupController struct { ... // Use to track the result of the deletion command. // Also used to track if controller crashes after the deletion command is triggered. deletingMapLock *sync.Mutex inProgressDeletingMap map[string]*DeletingStatus ... } ``` Go routine can add the error message to the `inProgressDeletingMap` if the command fails. Controller then can retry the command in the next reconciliation. ### Deleting/Error State with BackOff When a Backup is being deleted, indicated by the `DeletionTimestamp` in the CR, its state transitions should follow the diagram below. ``` # Normal Case Completed => Deleting => finalizer removed (CR isgone) # Command failure Case Completed => Deleting => Error (found the error message in the map) => Deleting (Retry the command) => finalizer removed (CR isgone) ``` Controller only triggers the deletion command if the state is not `Deleting`, and it updates the Backup's state to `Deleting` right after. To prevent the deletion command from being executed too frequently and to avoid rapid changes in status between `Deleting` and `Error`, **we implement a backoff strategy** with an initial delay of 10 seconds and a maximum delay of 1 minute before triggering the deletion command. ### Controller Crashes If controller crashes, the command doesn't finish and the map is cleaned up, the Backup could stuck in `Deleting` state forever. In this case, controller won't be able to aware the command is not running and won't even retry the command since the state is `Deleting`. Moreover, controller won't be able to dinstinguish the case of command not running or command is still running when there is no record in the map. Thus, we add the record with `state=Deleting` when the command is triggered. ``` type DeletingStatus struct { State longhorn.BackupState ErrorMessage string } type BackupController struct { ... // Use to track the result of the deletion command. // Also used to track if controller crashes after the deletion command is triggered. deletingMapLock *sync.Mutex inProgressDeletingMap map[string]*DeletingStatus ... } ``` **Controller further checks if there is a record in the map** when controller finds the Backup's state is `Deleting` and backup still exists in the backupstore. If there is no record in the map, it may indicate that the controller has crashed. In this case, the controller will update the status to `Error` and retry the command in the next reconciliation. The state transition should follow the diagram below ``` # Command failure Case Completed => Deleting => Error (found the error message in the map) => Deleting (Retry the command) => finalizer removed (CR isgone) # Controller crashes Completed => Deleting => Error (failed to find the record in the map) => Deleting (Retry the command) => finalizer removed (CR isgone) ``` ### Test plan You can use following commands to monitor the status. ``` $ watch -n 1 "kubectl get lhb -n longhorn-system -oyaml | grep -A 20 "status:"" $ watch -n 1 kubectl get lhb -n longhorn-system ``` #### Normal Case 1. Create a Volume 2. Write small data and then create a BackupA 3. Write large data (~2G) and then create a BackupB 4. Write small data and then create a Snapshot 5. Delete the BackupB(large data), at the same time, create a BackupC from the Snapshot(you can click from UI) 6. BackupC will be in `Pending` state with message (`waiting for backupB to be deleted`) 7. After BackupB is deleted, BackupC should be in progress. #### Error Case (use nfs) 1. Create a Volume 2. Write some data and then create a BackupA 3. Write some data and then create a sSnapshot 4. Exec into the backupstore pod and make the backup.cfg immutable ``` $ chattr +i backups/backup_backup-5640dfd33a054f98.cfg` ``` 5. Delete the BackupA, at the same time, create a BackupB from the Snapshot(you can click from UI) 6. BackupA will be in `Deleting` and `Error` state repeatedly to retry the deletion. When in `Error` state, it shows error message related to permission 7. BackupB will be `InProgress` when BackupA is in `Error` state. BackupB should be complete after awhile. 8. Remove the immutable, after a while, the BackupA should be in `Deleting` again and should be deleted successfully. #### Controller Crashes Case (use nfs) 1. Create a Volume 2. Write some data and then create a BackupA 3. Exec into the backupstore pod and make the backup.cfg immutable, example ``` $ chattr +i backups/backup_backup-5640dfd33a054f98.cfg` ``` 5. Delete the BackupA 6. BackupA will be in `Deleting` and `Error` state repeatedly to retry the deletion. When in `Error` state, it shows error message related to permission 7. When the BackupA is in `Deleting` state, delete the longhorn manager pod directly. (you can find the one doing the deleting with `Backup.Status.OwnerID`) 8. After the longhorn manager pod is recreated, the BackupA should turn into `Error` state with message `No deletion in progress record, retry the deletion command` 9. Then after a while the BackupA should be in `Deleting` again and should be deleted successfully after remove the immutable. ### Upgrade strategy No Need ## Note [optional] Additional notes. ================================================ FILE: enhancements/20240926-multiple-backup-targets-support.md ================================================ # Multiple Backup Targets Support ## Summary The current Longhorn only supports a single backup target for all volumes. The feature aims to provide multiple backup targets. ### Related Issues https://github.com/longhorn/longhorn/issues/2317 https://github.com/longhorn/longhorn/issues/5411 ## Motivation Users can choose backup targets to that they want to back up volumes by setting the backup target name of the volumes. ### Goals - There are different backup targets at the same time on Longhorn. - Backups can be synchronized from different remote backup targets. - Snapshots of a volume can be backed up to a specific remote backup target according to the backup target name of the volume. - A backup can be restored from its remote backup target to be a new volume. - Separate default backup target related settings `backup-target`, `backup-target-credential-secret`, and `backupstore-poll-interval` from global settings into another category. ### Non-goals [optional] - Backing up volumes to multiple backup targets concurrently is not a goal. ## Proposal 1. Allow users to create, delete or modify backup targets. - The backup target controller needs to synchronize the backups' information from the remote backup target and create `Backup` CR objects and `BackupVolume` CR objects in the cluster when the backup target is created. - When the backup target is deleted, backup target controller needs to delete the `Backup` CR and `BackupVolume` CR objects in the cluster related to the backup target, and backup data will be kept on the remote backup target. 2. Introduce a new field `BackupTarget` in `Status` of the `Backup` CRD to keep backup target information. - The field `BackupTarget:` is used to specify the backup target of the backup for the backup information from the `kubectl get` command and Longhorn RESTful APIs. 3. Introduce new fields `BackupVolume.Spec.BackupTargetName` and `BackupVolume.Spec.VolumeName` to keep information of the remote backup target and volume name. - Different backup targets might have volumes with the same names. The `BackupVolume` object name will be a randomly generated UUID with a prefix. It uses the `BackupTargetName` and `VolumeName` fields to specify the volume on the remote backup target. 4. Introduce new fields `BackupBackingImage.Spec.BackupTargetName` and `BackupBackingImage.Spec.BackingImage` to keep information of the remote backup target and backing image name. - Different backup targets might have backing images with the same names. The `BackupBackingImage` object name will be a randomly generated UUID with a prefix. It uses the `BackupTargetName` and `BackingImage` fields to specify the backing image on the remote backup target. 5. Introduce a new field `Volume.Spec.BackupTargetName` to mark the default backup target for storing the volume backups. 6. Not only the system backup will be handled by the default backup target but also the volumes without specifying the field `Spec.BackupTargetName`. 7. Support the parameter `backupTargetName` in the `StorageClass`. 8. Get default backup target related settings from the new `ConfigMap` `longhorn-default-backupstore` and remove them from the default settings. ### User Stories #### Default Backup Target The `default` backup target will be created automatically when a fresh installation, and it exists already when upgrading. If a volume doesn't have a specified backup target, the `default` backup target will be used. The `default` backup target can not be deleted by users. #### Create and Delete a new backup target Users need to create/modify backup targets on the page `Setting/Backup Target` as `Setting/Backing Image` via the Longhorn UI or by the manifest. After backup targets are created and synchronized, users can start to create a backup on these remote backup targets. Volume can not be created with specifying a non-existing or deleting backup target name. Users need to update the filed `Spec.BackupTargName` of the volumes manually after deleting the backup target. A warning or event should be shown if users try to delete a backup target still referenced by volumes or storage classes. ### User Experience In Detail #### Modify `default` Backup Targets With Helm The default backup target settings `backup-target`, `backup-target-credential-secret`, and `backupstore-poll-interval` will be moved to the new category `defaultBackupStoreSettings` from the category `defaultSettings` in the file `values.yaml`. - `defaultBackupStoreSettings.backupTarget` is used as `defaultSetting.backupTarget` to set up the default backup target URL. - `defaultBackupStoreSettings.backupTargetCredentialSecret` is used as `defaultSetting.backupTargetCredentialSecret` to set up the name of the default backup target secret. - `defaultBackupStoreSettings.pollInterval` is used as `defaultSetting.backupstorePollInterval` to set up how long to check the default backup target status. #### Create Or Modify Backup Targets By UI 1. There will be an empty default backup target created by Longhorn and users should set up the default backup target before creating a new backup target. 2. User can find the link `Backup Target` in drop down menu of `Setting`. 3. The page `Backup Target` would show backup target name, backup target URL, secret name (credential) for this backup target, poll interval, and availability status. 4. User can create a new backup target on the page `Backup Target` by clicking the button `Create`. User have to fill out the necessary item for the backup target name, and it can be optional for the backup target URL, secret name, and poll interval fields. 5. Users can modify a backup target information by clicking the button `Edit`. 6. Users can delete a backup target by clicking the button `Delete`. 7. Users can not create or modify a backup target with the `BackupTargetURL` that is the same to an existing backup target. 8. Users can not delete the default backup target directly. 9. Users can not modify the backup target name. 10. Users can not modify the `BackupTargetURL` of a backup target that is the same to another one in the cluster. #### Create Or Modify Backup Targets By CLI User can the bash command to create or modify a backup target such as below: ```bash cat <>new-backup-target.yaml apiVersion: longhorn.io/v1beta2 kind: BackupTarget metadata: name: azure-blob-server-001 namespace: longhorn-system spec: backupTargetURL: azblob://demo@core.windows.net/ credentialSecret: azblob-secret pollInterval: 4m30s EOF kubectl apply -f new-backup-target.yaml ``` ##### Backup Target Conditions There is only one backup target condition and one reason now. ```golang const ( BackupTargetConditionTypeUnavailable = "Unavailable" BackupTargetConditionReasonUnavailable = "Unavailable" ) ``` The condition `BackupTargetConditionTypeUnavailable` will be set to true if the backup target URL is empty or if retrieving backup information from the remote backup target fails due to issues such as: - Backup list command timeout. - Invalid backup target URL. - Invalid secret data for the backup target. - Protocol version of the remote NFS server is not supported. Longhorn support NFSv4, NFSv4.1, and NFSv4.2 now. - Permission of the remote data path, directory, bucket or container, is not allowed to read/write. - The network is disconnected for a while. #### Create A Backup To Remote Backup Targets By UI After users click the button `Create Backup` on the `Volume Details` page, the backup will be stored on the remote backup target set in the `BackupTargetName` filed of the volume. The backup volume will be created and displayed on the `Backup` page if it is the first backup of the volume. On the `Backup` page, the backup targets are displayed first if one or more backup volumes exist. Clicking a backup target will expand or collapse the list to show the associated backup volumes. #### Create A Backup To Remote Backup Targets By CLI 1. Create a snapshot of the volume. 2. Create a backup referring to store the snapshot by the manifest as below: ```yaml apiVersion: longhorn.io/v1beta2 kind: Backup metadata: labels: backup-volume: pvc-xxxxxxxx-9ab9-4055-9eb4-558999b09a11 name: backup-test-001 namespace: longhorn-system spec: labels: longhorn.io/volume-access-mode: rwo snapshotName: xxxxxxxx-9a69-4ede-ab88-c9853459462c ``` 3. The `status.backupTarget` field will be assigned to match the `spec.backupTargetName` field in the volume of the snapshot after the backup is completed. The field `status.backupTarget` displays the backup target name in the column information of the backup when users run the `kubectl get` command, and populates the backup target name in the backup information returned by the Longhorn RESTful APIs. #### Create A Backup To Remote Backup Targets By Recurring Jobs It will store snapshots to the appointed backup target set in the `spec.backupTargetName` of the volume when the recurring job is running. #### Restore From A Backup From A Remote Backup Target Restoring from a backup will behave as before. Choose a backup and do the `Restore` operation in drop down menu on UI. #### Create A Disaster Recovery Volume From A Remote Backup Target ![dr_volume](./image/mbt-dr-001.png) Creating a DR volume can be from any backups of valid backup targets in the cluster. As the graph above, the DR volume `DRvolX` is created from the backup of the `default` backup target and the DR volume `DRvolY` can be from the backup of the backup target `BT1`. If users want to create all DR volumes of the cluster 2 in the cluster 1, users need to create all backup targets that volumes of the cluster 2 using. If users only want to create a DR volume for the volume `volA2` in the cluster 2, user only need to create the backup target `BT1` in the cluster 1 and Longhorn will poll the incremental backups on the remote backup target B. #### Create A Backup Of The Backing Image To Remote Backup Targets Allow users to choose a backup target when creating a backup for a backing image. This behavior differs from the backup operation of a volume. The volume custom resource uses a fixed field, `Spec.BackupTargetName`, to determine the remote backup target when creating incremental backups. In contrast, a backing image typically has only one backup on the remote backup target, so it is unnecessary to include a `Spec.BackupTargetName` field for storing incremental backups of a backing image on the same backup target. ##### Create A Disaster Recovery Volume From A Remote Backup Target By UI 1. In the cluster A, make sure the original volume X has a backup created or has recurring backups scheduled. 2. In `Backup` page of cluster B, choose the backup volume X, then create disaster recovery volume Y. 3. Longhorn will automatically attach the DR volume Y to a random node. Then Longhorn will start polling for the last backup of volume X, and incrementally restore it to the volume Y. ### API changes - Introduce new APIs `BackupTargetCreate`,`BackupTargetDelete`, `BackupTargetUpdate` and `BackupTargetGet`: | API | Input | Output | Comments | HTTP Endpoint | | --- | --- | --- | --- | --- | | Create | name, backupTargetURL, credentialSecret string, pollInterval time.Duration | err error | Create a new backup target and start to synchronize data | **POST** `/v1/backuptargets/` | | Delete | name string | err error | Remove a backup target and its related backup volume and backup CR objects in the cluster | **DELETE** `/v1/backuptargets/{backupTargetName}` | | Update | backupTargetURL, credentialSecret string, pollInterval time.Duration | err error | Update the backup targets information | **POST** `/v1/backuptargets/{backupTargetName}?action=backupTargetUpdate` | | Get | | backupTarget BackupTarget, err error | Get the backup targets information | **GET** `/v1/backuptargets/{backupTargetName}` | - Modify the APIs `BackupVolumeGet`, `BackupVolumeList`: - Add a new field `BackupTargetName` and `VolumeName` in returning `BackupVolume` information for all backup targets that have this backup volume. - Modify the APIs `BackupGet`: - Add a new field `BackupTargetName` in returning `Backup` information. - Modify the API `SnapshotBackup`: - Add a new field `BackupTargetName` in the input `SnapshotInput` and returning `Backup` information. - Modify the API `BackupBackingImageCreate`, `BackupBackingImageGet` and `BackupBackingImageList`: - Add new fields `BackingImage` and `BackupTargetName` in the input `BackupBackingImage`. ```golang type BackupBackingImage struct { client.Resource Name string `json:"name"` BackingImage string `json:"backingImage"` BackupTargetName string `json:"backupTargetName"` ... } ``` ## Design ### Implementation Overview #### Custom Resource Definitions 1. Modify the Backup CRD `backups.longhorn.io` to add a new field `BackupTarget` in `Status` for creating a backup. And add a label `LonghornLabelBackupTarget = "backup-target"` for resource listing and `OwnerReferences` for its corresponding backup volume. ```golang type BackupStatus struct { ... // The backup target name. BackupTarget string `json:"backupTarget"` } ``` ```yaml metadata: labels: longhorn.io/backup-target: the backup target name (string) longhorn.io/backup-volume: the volume name (string) OwnerReferences: GetOwnerReferencesForBackupVolume(string), name: the backup name. (string) status: backupTarget: the backup target Name. (string) ... ``` 2. Modify the Backup Volume CRD `backupvolumes.longhorn.io` to add new fields `BackupTargetName` and `VolumeName` in `Spec`, and add labels `LonghornLabelBackupTarget` and `LonghornLabelBackupVolume` for resource listing and `OwnerReferences` for its corresponding backup get. ```golang type BackupVolumeSpec struct { ... SyncRequestedAt metav1.Time `json:"syncRequestedAt"` // The backup target name that the backup volume was synced. BackupTargetName string `json:"backupTargetName"` // The volume name that the backup volume was used to backup. VolumeName string `json:"volumeName"` } ``` ```yaml metadata: labels: longhorn.io/backup-target: the backup target name (string) longhorn.io/backup-volume: the volume name (string) OwnerReferences: GetOwnerReferencesForBackupTarget(string), name: the backup volume name. (string) spec: backupTargetName: the backup target Name. (string) volumeName: the volume name (string) ... ``` 3. Modify the Backup Backing Image CRD `backupbackingimages.longhorn.io` to add new fields `BackingImage` and `BackupTargetName` in `Spec`, and add labels `LonghornLabelBackupTarget = "backup-target"` and `LonghornLabelBackingImage = "backing-image"`. ```golang type BackupBackingImageSpec struct { ... // The backing image name. BackingImage string `json:"backingImage"` // The backup target name. BackupTargetName string `json:"backupTargetName"` } ``` ```yaml metadata: labels: longhorn.io/backing-image: the backing image name (string) longhorn.io/backup-target: the backup target name (string) name: the backup backing image name. (string) spec: backingImage: the backing image name (string) backupTargetName: the backup target Name. (string) ... ``` 4. Modify the Volume CRD `volumes.longhorn.io` to add a new field `BackupTargetName` in `Spec` for a default backup target. ```golang type VolumeSpec struct { // The default backup target name when a backup is created from the volume. BackupTargetName int64 `json:"backupTargetName"` } ``` ```yaml metadata: name: the volume name. (string) spec: backupTargetName: the backup target Name. (string) ``` The `Spec.BackupTargetName` field will default to the backup target when a volume is created with an empty backup target name, and volume backups will be stored in the default backup target. If the `Spec.BackupTargetName` field is updated to a new backup target, subsequent backups will be stored in the new target. #### Backup Related Controllers - Modify the backup target controller to allow adding and deleting an extra backup target. - Create pulling backup volume CR objects with a random backup volume name, for example, `bv-xxxxxx-a12345-b12345-xxxxxx` and filling in the fields `Spec.BackupTargetName` and `Spec.VolumeName`, and corresponding labels from the backup target if it does not exist according to the fields `Spec.BackupTargetName` and `Spec.VolumeName`. - Create pulling backup backing images with a random backup volume name, for example, `bbi-xxxxxx-a12345-b12345-xxxxxx` from the backup target with filling in the fields `Spec.BackupTargetName` and `Spec.BackingImage`, and corresponding labels if not existing. - Remove a backup volume or backup backing image CR object if the corresponding backup target is deleting or the URL of the corresponding backup target is empty. - Modify the backup volume controller to tell the backup volume belongs to which backup target and synchronize the backups of the backup volume from remote backup target. - Create pulling backups from the backup target with the field `Status.BackupTarget` of the backup target. - Clean up backup CR objects in the cluster by the volume name and backup target name when deleting a backup volume or the backup target URL is empty. - Modify backup backing image controller - Start backing up a backing image with the given backup target name. #### Delete An Extra Backup Target - Deleting the `default` backup target is not allowed. - When DR volumes exist, deleting any backup target is not allowed. - When deleting a backup target is allowed, the backup volumes and backups associated with this target will be cleaned up (the remote backup data will remain stored). This will not affect backup volumes and backups of other backup targets. - Volumes where the `Spec.BackupTargetName` field matches the name of the deleted backup target will retain their original value, but creating a new backup will fail. #### Handle Backup Target URL Changed The behavior will be the same as current implementation. - When an extra backup target URL is valid, the backup target controller will synchronize backup volumes and backups information from the remote backup target to the cluster. - When emptying the extra backup target URL, the backup target controller will clean up backup volumes and backups information in the cluster. This will not affect backup volumes and backups of other backup targets. - When an extra backup target URL is modified from a valid URL to an invalid URL, the backup target controller will set this backup target unavailable and unavailable reason in the `Status.Condition`, and skip the synchronization with the remote backup target. Backup volumes and backups synchronized from the previous valid URL will not be cleaned up. #### Add A New Category For Default Backup Settings 1. Separate the default backup related settings from default settings into the new category in Helm `values.yaml` file: ```yaml defaultBackupStoreSettings: # -- Endpoint used to access the default backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") backupTarget: ~ # -- Name of the Kubernetes secret associated with the default backup target. backupTargetCredentialSecret: ~ # -- Number of seconds that Longhorn waits before checking the default backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. pollInterval: ~ ``` 2. Create a new `ConfigMap` `default-backupstore.yaml` to store these settings in Helm chart: ```yaml apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-backupstore namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} data: default-backupstore.yaml: |- {{- if not (kindIs "invalid" .Values.defaultBackupStore.backupTarget) }} backup-target: {{ .Values.defaultBackupStore.backupTarget }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultBackupStore.backupTargetCredentialSecret) }} backup-target-credential-secret: {{ .Values.defaultBackupStore.backupTargetCredentialSecret }} {{- end }} {{- if not (kindIs "invalid" .Values.defaultBackupStore.pollInterval) }} backupstore-poll-interval: {{ .Values.defaultBackupStore.pollInterval }} {{- end }} ``` 3. The Kubernetes `ConfigMap` controller will monitor the `ConfigMap` `longhorn-default-backupstore` and update the default backup target if settings are modified. ```go func (kc *KubernetesConfigMapController) reconcile(namespace, cfmName string) error { ... case types.DefaultDefaultSettingConfigMapName: if err := kc.ds.UpdateCustomizedSettings(nil); err != nil { return errors.Wrap(err, "failed to update built-in settings with customized values") } case types.DefaultDefaultBackupStoreConfigMapName: if err := kc.ds.CreateOrUpdateDefaultBackupStore(); err != nil { return errors.Wrap(err, "failed to crerate or update default backup target with customized values") } } } ``` #### Uninstallation When uninstalling Longhorn: 1. Empty the URL of all backup targets to safely clean up backup volume and backup custom resources in the cluster. 2. Delete all backup target custom resources in the cluster. ### Validating And Mutating Webhook - If the `Spec.BackupTargetName` field of the Volume CR object is empty, populate it with the default backup target name. - The volume validator will check if the backup target exists after the volume is created or the field `Spec.BackupTargetName` is updated. ### Test plan #### Update The Default Backup Target 1. Install with Helm. 2. Update the default backup setting `defaultBackupStore.backupTarget` with a valid URL of a remote backup target. 3. Update the default backup setting `defaultBackupStore.backupTargetCredentialSecret` if needed. 4. Check if the default backup target is available. 5. Update the default backup setting `defaultBackupStore.pollInterval` and it succeeds. #### Create A Volume By A Storage Class For The Parameter `backupTargetName` 1. Create a Storage Class A without the parameter `backupTargetName`. 2. Create a volume A with the Storage Class A successfully and the field `spec.backupTargetName` of volume A is `default`. 3. Create a Storage Class B with the parameter `backupTargetName: backupTargetB` (the backup target B exists.) 4. Create a volume B with the Storage Class B successfully and the field `spec.backupTargetName` of volume B is `backupTargetB`. 5. Create a Storage Class C with the parameter `backupTargetName: backupTargetC` (the backup target C does not exist.) 6. Creating a volume C is not allowed. 7. Create a volume D with the field `spec.backupTargetName` is `backupTargetB`. 8. Create a volume D successfully and the field `spec.backupTargetName` of volume D is `backupTargetB`. #### Create And Restore A Backup 1. Set up the default backup target. 2. Create a backup to the default backup target, and it succeeds. 3. Create an extra backup target A which has existing backups. 4. Backups of the default backup target will not be deleted. 5. Existing backups on the extra backup target A can be synchronized back to the cluster. 6. Create a backup to extra backup target A (by setting the `Spec.BackupTargetName` field in the volume first), and it succeeds. 7. Restore a backup from the default backup target, and it succeeds and data is correct. 8. Restore a backup from the extra backup target A, and it succeeds and data is correct. #### Create And Restore A Backing Image 1. Set up the default backup target and create an extra backup target A. 2. Create a backing image. 3. Create a backup of the backing image to the default backup target, and it succeeds. 4. Create a backup of the backing image to extra backup target A, and it succeeds. 5. Restore the backup of the backing image from the default backup target, and it succeeds and data is correct. 6. Restore the backup of the backing image from the extra backup target A, and it succeeds and data is correct. #### Create A DR Volume 1. Prepare two clusters A and B with Longhorn installed. 2. Set up the default backup target of two clusters with the same remote backup target. 3. Create the volume A and create a backup of the volume A in the cluster A. 4. In the cluster B, create a DR volume after the backup A is synchronized. 5. Write data to the volume A and create a new backup B in the cluster A. 6. Modifying/Deleting the default backup target URL is not allowed. 7. Check if the DR volume will synchronize the data in the cluster B. #### Create A DR Volume Through Another Backup Target 1. Prepare two clusters A and B with Longhorn installed. 2. Set up an extra backup target BT of two clusters with the same remote backup target. 3. Create the volume A and create a backup of the volume A in the cluster A. 4. In the cluster B, create a DR volume after the backup A is synchronized. 5. Write data to the volume A and create a new backup B in the cluster A. 6. Modifying/Deleting the default backup target URL is not allowed. 7. Check if the DR volume will synchronize the data in the cluster B. #### Create And Restore A System Backup 1. Set up the default backup target and an extra backup target A. 2. Create a volume A with the default backup target name and a volume B with the extra backup target name A. 3. Write some data to volume A and volume B. 4. Create a system backup, and it succeeds. (the system backup and the volume A is stored on the default backup target, and the volume B is stored on the backup target A) 5. Delete the backup target A. 6. Restore the system backup, and it succeeds. 7. The backup target A should be restored. #### Modify The Backup Target URL 1. Set up the default backup target and add an extra backup target A which has existing backups. 2. Existing backups on the extra backup target A can be synchronized back to the cluster. 3. Modify the extra backup target A URL to another valid URL of backup target B and related backup volume and backup custom resources will be synchronized correctly from the remote backup target B. 4. Modify the extra backup target A URL to an invalid URL. 5. The extra backup target A become unavailable and synchronization will be skipped. (related backup volume and backup custom resources of the backup target B will not be cleaned up.) 6. Empty the extra backup target A URL and related backup volume and backup custom resources of the backup target B will be cleaned up. 7. Backup volume and backup custom resources of the default backup target will not be affected. #### Delete A Backup Target 1. Set up the default backup target and add an extra backup target B which has existing backups. 2. Deleting the default backup target is not allowed. 3. Existing backups on the extra backup target B can be synchronized back to the cluster. 4. Create a backup to the default backup target, and it succeeds. 5. Create a backup to extra backup target B (by setting the `Spec.BackupTargetName` field in the volume first), and it succeeds. 6. Delete the extra backup target and related backup volume and backup custom resources will be cleaned up. 7. Backup volume and backup custom resources of the default backup target will not be affected. 8. Add the extra backup target back and related backup volume and backup custom resources will be synchronized correctly. #### Uninstall 1. Set up the default backup target and an extra backup target A. 2. Create a volume A with the default backup target name and a volume B with the extra backup target name A. 3. Create backups for the volume A and B and wait for backups completion. 4. Uninstall Longhorn 5. URLs of backup targets are empty successfully. 6. Backup targets are deleted successfully. 7. Uninstall successfully. ### Upgrade strategy The `default` backup target will not be deleted on old versions Longhorn - Fill in the field `Status.BackupTarget` for existing Backup CR objects and labels with corresponding values of the `default` backup target. - Fill in the fields `Spec.BackupTargetName` and `Spec.VolumeName` and labels for existing `BackupVolume` CR objects with corresponding values of the `default` backup target and volume. - Fill in the fields `Spec.BackupTargetName` and `Spec.BackingImage` and labels for existing `BackupBackingImage` CR objects with corresponding values of the `default` backup target and backing image. - Fill in the fields `Spec.BackupTargetName` for existing `Volume` CR objects with corresponding values of the `default` backup target. ## Note [optional] `None` ================================================ FILE: enhancements/20241003-improve-pulling-backups-from-the-backup-target.md ================================================ # Improve Pulling Backups From The Remote Backup Target ## Summary If the NFS server experiences a brief downtime, the backup data stored on the NFS backup target may be deleted. This occurs because, when the NFS server comes back online, attempts to access the backup target might initially return an empty response. This situation indicates potential instability or delay in data availability following the server's recovery, which could impact the reliability of the related backup resources (`BackupVolume`, `BackupBackingImage`, `SystemBackup` and `Backup`) management in the backup target and backup volume controllers. Proper data management should be taken to handle the initial empty responses until full data access is restored. ### Related Issues https://github.com/longhorn/longhorn/issues/9530 ## Motivation ### Goals Delete only the backup-related resources in the cluster and do not delete anything on the backup target if there is a discrepancy between the backup information in the cluster and that on the backup target. ### Non-goals [optional] `None` ## Proposal - Modify the backup target controller to add the label `DeleteCustomResourceOnly` to the `BackupVolume`, `BackupBackingImage`, and `SystemBackup` resources when no such corresponding backups' data on the remote backup target. - `BackupVolume`: 1. Modify the backup volume controller to check whether the label `DeleteCustomResourceOnly` is set when deleting the `BackupVolume` resource. 2. Modify the backup volume controller to add the label `DeleteCustomResourceOnly` to the `Backup` resource when no such backup on the remote backup target. - `BackupBackingImage`: - Modify the backup backing image controller to check whether the label `DeleteCustomResourceOnly` is set when deleting the `BackupBackingImage` resource. - `SystemBackup`: - Modify the system backup controller to check whether the label `DeleteCustomResourceOnly` is set when deleting the `SystemBackup` resource. - `Backup`: - Modify the backup controller to check whether the label `DeleteCustomResourceOnly` is set when deleting the `Backup` resource. ### User Stories #### Empty Response from Remote NFS Backup Target The remote NFS server might initially return an empty response after a short downtime. Therefore, the backup data on the remote NFS server will be deleted unexpectedly with the synchronization mechanism of the backup target controller if the response is empty and the remote backup target is available. After this enhancement, only the related backup resources will be deleted and the remote backup data will not be deleted with the synchronization mechanism. #### Race Condition between Related Backup Controllers A race condition which will unexpectedly delete the backup data on the remote backup target exists not only between backup target and backup volume controllers but also between backup volume and backup controllers. It will happen when: 1. Users empty the backup target URL. 2. The backup target controller start to reconcile the backup target and try to delete the backup volume resources of the backup target. 3. The backup volume resources are marked as deleted. 4. Users set the backup target URL as the previous one. 5. The backup volume controller start to reconcile the deleting backup volumes and the backup target becomes available. 6. Because the backup target is available, the backup volume controller will delete the backup volume resources with the backups' data on the remote backup target. What users expect will be: 1. Emptying the backup target URL will only delete related backups' resources in the cluster 2. Re-set the backup target URL as the previous one and the related backups' resource will be re-created in the cluster 3. The backup data on the remote backup target will not be deleted. After this enhancement, the race condition is not eliminated, but it will not delete the backup data on the remote backup target. ### User Experience In Detail The user experience will be the same after this enhancement. - Set up and unset a backup target URL (and the corresponding credential) - When a backup target is not available, Longhorn will try to clean up all related backup resources (`BackupVolume`, `BackupBackingImage`, `SystemBackup` and `Backup`) of the backup target in the cluster. - When a backup target is available, Longhorn will try to compare related backup resources to the remote backup target: - Create related backup resources if they exist on the remote backup target and does not exist in the cluster. - Delete related backup resources if they exist in the cluster and does not exist on the remote backup target, and Longhorn will not delete remote backups' data on the backup tart. - Create and delete a backup: - Users can create a backup with the `kubectl` command or Longhorn UI, and creating a corresponding backup volume will be triggered if it is the first backup in the cluster. - Users can delete a backup with the `kubectl` command or Longhorn UI, - The `Backup` resource in the cluster will be deleted. - The backup data on the remote backup target will be deleted when the backup target and corresponding backup volume are available, and the backup is not used for restoring a volume. - Create and delete a backup backing image or a system backup: - Users can create a backup backing image or a system backup with the `kubectl` command or Longhorn UI. - Users can delete a backup backing image or a system backup with the `kubectl` command or Longhorn UI, - The `BackupBackingImage` or `SystemBackup` resource in the cluster will be deleted. - The related backup data on the remote backup target will be deleted when the backup target is available - Delete a backup volume: - Users can delete a backup volume with the `kubectl` command or Longhorn UI, - The `BackupVolume` resource in the cluster will be deleted. - The related backups' data on the remote backup target will be deleted when the backup target is available. ### API changes `None` ## Design ### Implementation Overview These labels will only be added when the backup target is reconciled and the synchronization of related backup resources begins. In other words, when the corresponding data is not found in the remote backup target, or the backup target is no longer available, Longhorn will try to remove the CR and skip the actual data deletion as the aggressive data cleanup is too dangerous. 1. Add labels `DeleteCustomResourceOnly` in the file `types/types.go`: ```golang ... ConfigMapResourceVersionKey = "configmap-resource-version" UpdateSettingFromLonghorn = "update-setting-from-longhorn" DeleteCustomResourceOnly = "delete-custom-resource-only" ... ``` 2. At the existed `backup_target_controller`: Add corresponding labels for cleaning up the resources when synchronizing the backups' information to the remote backup target. - For `BackupVolume`: ```golang ... func (btc *BackupTargetController) syncBackupVolume(backupStoreBackupVolumeNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { ... for backupVolumeName := range backupVolumesToDelete { // Add the label `DeleteCustomResourceOnly` and update the `BackupVolume` resource if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil { return errors.Wrapf(err, "failed to delete backup volume %s from cluster", backupVolumeName) } } ... } ... func (btc *BackupTargetController) cleanupBackupVolumes() error { ... for backupVolumeName := range clusterBackupVolumes { // Add the label `DeleteCustomResourceOnly` and update the `BackupVolume` resource if err = btc.ds.DeleteBackupVolume(backupVolumeName); err != nil && !apierrors.IsNotFound(err) { errs = append(errs, err.Error()) continue } } ... } ... ``` - For `BackupBackingImage`: ```golang ... func (btc *BackupTargetController) syncBackupBackingImage(backupStoreBackingImageNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { ... for backupBackingImageName := range backupBackingImagesToDelete { // Add the label `DeleteCustomResourceOnly` and update the `BackupBackingImageName` resource if err = btc.ds.DeleteBackupBackingImage(backupBackingImageName); err != nil { return errors.Wrapf(err, "failed to delete backup backing image %s from cluster", backupBackingImageName) } } ... } ... func (btc *BackupTargetController) cleanupBackupBackingImages() error { ... for backupBackingImageName := range clusterBackupBackingImages { // Add the label `DeleteCustomResourceOnly` and update the `BackupBackingImage` resource if err = btc.ds.DeleteBackupBackingImage(backupBackingImageName); err != nil && !apierrors.IsNotFound(err) { errs = append(errs, err.Error()) continue } } ... } ... ``` - For `SystemBackup`: ```golang ... func (btc *BackupTargetController) syncSystemBackup(backupStoreBackingImageNames []string, syncTime metav1.Time, log logrus.FieldLogger) error { ... delSystemBackupsInCluster := clusterReadySystemBackupNames.Difference(backupstoreSystemBackupNames) for name := range delSystemBackupsInCluster { // Add the label `DeleteCustomResourceOnly` and update the `SystemBackup` resource if err = btc.ds.DeleteSystemBackup(name); err != nil { return errors.Wrapf(err, "failed to delete SystemBackup %v not exist in backupstore", name) } } ... } ... func (btc *BackupTargetController) cleanupSystemBackups() error { ... for systemBackup := range systemBackups { // Add the label `DeleteCustomResourceOnly` and update the `SystemBackup` resource if err = btc.ds.DeleteSystemBackup(systemBackup); err != nil && !apierrors.IsNotFound(err) { errs = append(errs, err.Error()) continue } } ... } ... ``` 3. At the existed `backup_volume_controller`: - Add the label `DeleteCustomResourceOnly` for cleaning up the backup resource when synchronizing the backups' information to the remote backup target - Check if it needs to delete the remote backup volume data when a `BackupVolume` resource is deleting. ```golang func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error) { ... if !backupVolume.DeletionTimestamp.IsZero() { // Get the flag `needCleanupRemoteData` from the label `DeleteCustomResourceOnly` to check if it needs to delete remote backup volume data // if the flag `needCleanupRemoteData` is false, add the label `DeleteCustomResourceOnly` to all backups of the backup volume and update the `Backup` resource if err := bvc.ds.DeleteAllBackupsForBackupVolume(backupVolumeName); err != nil { return errors.Wrap(err, "failed to delete backups") } // Modify the judgement to `if needCleanupRemoteData && backupTarget.Spec.BackupTargetURL != "" {` if backupTarget.Spec.BackupTargetURL != "" { ... if err := backupTargetClient.BackupVolumeDelete(backupTargetClient.URL, backupVolumeName, backupTargetClient.Credential); err != nil { return errors.Wrap(err, "failed to delete remote backup volume") } } ... } ... backupsToDelete := clustersSet.Difference(backupStoreBackups) for backupName := range backupsToDelete { // Add the label `DeleteBackupResourceOnly` and update the `Backup` resource if err = bvc.ds.DeleteBackup(backupName); err != nil { return errors.Wrapf(err, "failed to delete backup %s from cluster", backupName) } } ... } ``` 4. At the existed `backup_controller`: - Check if it needs to delete the remote backup data when a `Backup` resource is deleting. ```golang func (bc *BackupController) reconcile(backupName string) (err error) { ... if !backup.DeletionTimestamp.IsZero() { ... // Get the flag `needCleanupRemoteData` from the label `DeleteCustomResourceOnly` to check if it needs to delete remote backup data // Modify the judgement to `if needCleanupRemoteData && backupTarget.Spec.BackupTargetURL != "" &&` if backupTarget.Spec.BackupTargetURL != "" && backupVolume != nil && backupVolume.DeletionTimestamp == nil { ... if err := backupTargetClient.BackupDelete(backupURL, backupTargetClient.Credential); err != nil { return errors.Wrap(err, "failed to delete remote backup") } } } ... } ``` 5. At the existed `backup_backing_image_controller`: - Check if it needs to delete the remote backup backing image data when a `BackupBackingImage` resource is deleting. ```golang func (bc *BackupBackingImageController) reconcile(backupBackingImageName string) (err error) { ... if !bbi.DeletionTimestamp.IsZero() { // Get the flag `needCleanupRemoteData` from the label `DeleteCustomResourceOnly` to check if it needs to delete remote backup backing image data // Modify the judgement to `if needCleanupRemoteData && backupTarget.Spec.BackupTargetURL != "" {' if backupTarget.Spec.BackupTargetURL != "" { ... if err := backupTargetClient.BackupBackingImageDelete(backupURL); err != nil { return errors.Wrap(err, "failed to delete remote backup backing image") } } ... } ``` 6. At the existed `system_backup_controller`: - Check if it needs to delete the remote system backup data when a `SystemBackup` resource is deleting. ```golang func cleanupRemoteSystemBackupFiles(systemBackup *longhorn.SystemBackup, backupTargetClient engineapi.SystemBackupOperationInterface, log logrus.FieldLogger) { ... // Get the flag `needCleanupRemoteData` from the label `DeleteCustomResourceOnly` to check if it needs to delete remote system backup data if !needCleanupRemoteData { return } } ``` ### Test plan 1. Preparing a Linux host with NFS service 1. Set the backup target URL with this NFS service. 1. Creating three volume backups. 1. Setting `Backupstore Poll Interval` as 10 seconds 1. On the Linux host with NFS service, pointing `KUBECONFIG` to the Longhorn control plane IP address. 1. On the Linux host with NFS service, executing [monitor_nfs_longhorn_twice.sh](https://github.com/WebberHuang1118/misc-tools/blob/main/backups/monitor_nfs_longhorn_twice.sh), which will perform: 1. Stopping the NFS Service 1. Checking Failed to get info from backup store is shown twice on LH manager log 1. Starting the NFS Service around 59 seconds later 1. Checking if all LH backups are deleted (both in the backup target and the LH backup CR) ### Upgrade strategy `None` ## Note [optional] `None` ================================================ FILE: enhancements/20241028-auto-salvage-support-for-v2-volumes.md ================================================ # Auto-salvage Support For V2 Volumes ## Summary This document proposes extending Longhorn's auto-salvage feature to support v2 volumes. Currently, auto-salvage automatically recovers v1 volumes when all its replicas fail. This proposal aims to provide the same functionality for v2 volumes, improving data availability and reducing operational overhead. ### Related Issues https://github.com/longhorn/longhorn/issues/8430 ## Motivation ### Goals - **Improve Data Availability For V2 Volumes:** Auto-salvage ensures v2 volumes automatically recovers from replica failures, minimize downtime. - **Reduce Operational Overhead:** Automating recovery, freeing user from manual intervention during failures. ### Non-goals [optional] - `None` ## Proposal ### User Stories #### Story 1: Auto-salvage V2 Volumes When All Replicas Failed As a Longhorn user, I want Longhorn to automatically salvage a v2 volume when all its replicas fail, similar to the existing functionality for v1 volumes. - **Before:** Manual intervention was required to recover a faulted v2 volume when all replicas failed. - **After:** With auto-salvage enabled, Longhorn will attempt to salvage usable replicas and bring the v2 volume back online automatically. ### User Experience In Detail - **Auto-salvage:** When the setting is enabled, if all replicas for a volume fail, Longhorn attempts salvage usable replicas to recover the volume state. - **Volume Trim Operation Blocking:** For degraded v2 volumes, filesystem trim operation will be blocked to preserve a reliable volume head size for identifying usable replica candidates. ### API changes #### SPDK RPC Protobuf - Introduce `salvage_requested` boolean field in the `SpdkInstanceSpec` message. which is passed to the instance GRPC server during engine instance creation. ```go message SpdkInstanceSpec { map replica_address_map = 1; string disk_name = 2; string disk_uuid = 3; uint64 size = 4; bool expose_required = 5; string frontend = 6; bool salvage_requested = 7; } ``` - Introduce `salvage_requested` boolean field in the `EngineCreateRequest` message to pass to the SPDK server during engine creation. ```go message EngineCreateRequest { string name = 1; string volume_name = 2; uint64 spec_size = 3; map replica_address_map = 4; string frontend = 5; int32 port_count = 6; bool upgrade_required = 7; string initiator_address = 8; string target_address = 9; bool salvage_requested = 10; } ``` ## Design > **Note:** > The design applies only to v2 volumes. The v1 volume candidate selection remains unchange. ### Failed Usable Replica Filtering When `EngineCreate()` is called with `salvage_requested` set to `true`, the SPDK server retrieves `Replica` from SPDK server cache. We assume that all remaining replicas' lvols are identical and their lvol heads contain that latest data. The lvol head with the largest size is assumed to hold the most recent, valid data since larger size should indicates that it includes the latest writes or updates. Therefore, the head sizes of all replicas are sorted, and replicas with head sizes different from the largest in the sorted list are excluded from the candidate list. ### Engine Creation During SPDK replica creation, if the replica already exists, the lvol bdev creation process is skipped. Instead, `Replica.construct()` is called to build the `Replica` object with existing lvol. ### Volume Filesystem Trim Operation Blocking Block filesystem trim operation on degraded v2 volumes to preserve the lvol head size, ensuring reliable salvage replica candidate selection. ### Test plan 1. **Feature Testing:** 1. Use existing robot/integration test cases (cluster reboot, node powerdown, network disconnection, etc.) to verify auto-salvage functionality for v2 volumes after replica failures. 1. Introduce new robot test cases to verify that trim operations are blocked for degraded v2 volumes. 1. **Regression Testing:** Ensure v1 volume auto-salvage and volume trim operation remains unaffected by the this extended feature implementation. ### Upgrade strategy No specific upgrade strategy is required for this feature extension. ## Note [optional] None ================================================ FILE: enhancements/20241111-rwx-resize.md ================================================ # Title RWX Resize ## Summary Expansion of volumes requires resizing the filesystem inside the block device /dev/longhorn/\ after the block device is grown. In the case of an RWX volume, that is on the node where the share-manager pod has it mounted. But the CSI layer sends the `NodeExpandVolume` RPC request to the CSI plugin on the node(s) where the volume is attached, which might not include the mount node. If it doesn't, the resize does not take effect. This enhancement is meant to ensure that the CSI request or some equivalent is forwarded to the share-manager node. ### Related Issues [BUG] RWX expansion fails. Fail to resize RWX PVC at filesystem resizing step [#9736](https://github.com/longhorn/longhorn/issues/9736) ## Motivation ### Goals Resize of RWX volume should succeed regardless of location of share-manager and workload pods. This should also take care of the manual resize step documented in https://longhorn.io/docs/1.7.2/nodes-and-volumes/volumes/expansion/#rwx-volume, and allow fully automated expansion of RWX volumes without scale-down, resolving `[FEATURE] Automatic online RWX volume expansion` [#8118](https://github.com/longhorn/longhorn/issues/8118) ### Non-goals [optional] ## Proposal A csi-plugin can detect that the device path does not exist locally. But it doesn't really have to, because it can tell that the volume is RWX. It can delegate the resize to the owning share manager in all cases, even on the local node. Implement a "FilesystemResize" RPC call in the share-manager, much like its "FilesystemTrim", and let the csi-plugin refer the resize action to it. ### User Stories There's no extra work for the user. In fact, this should make an existing feature easier to use. However, the user must have restarted the share-manager pod after upgrade to the release with this feature, to get an image that supports the RPC call. ### API changes - Add a FilesystemResize method to share-manager RPC protocol ### Alternatives It might be tempting to find a way to forward the request to the longhorn-csi-plugin on the correct node. If we handle the expansion in share manager we must: 1. re-implement the logic 2. restart the share-manager to have new logic If we could handle the expansion by forwarding to correct csi-plugin, we would need to: 1. figure out how make csi-plugin act as a CSI RPC client and forward the request payload 2. OR send some other message, and allow csi-plugin to have kubectl client so it can fetch the secret passphrase for crypto resize (see below) 3. restart the csi-plugin, which happens automatically on upgrade Unfortunately, that won't work. The csi-plugin has no idea of the actual filesystem type. The fields of the `NodeExpandVolume` request are derived from the mount it knows about. On the csi-plugin pod, `mount -l` shows ``` 10.43.248.207:/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b on /var/lib/kubelet/plugins/kubernetes.io/csi/driver.longhorn.io/46e93d5a9308ccb4b10edb9f7d2e362f5e6f45f125927196166405a669c36d2b/globalmount type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,softerr,softreval,noresvport,proto=tcp,timeo=600,retrans=5,sec=sys,clientaddr=24.144.89.5,local_lock=none,addr=10.43.248.207) 10.43.248.207:/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b on /var/lib/kubelet/pods/44f0a9cc-6995-4fa9-a914-ca0748793315/volumes/kubernetes.io~csi/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b/mount type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,softerr,softreval,noresvport,proto=tcp,timeo=600,retrans=5,sec=sys,clientaddr=24.144.89.5,local_lock=none,addr=10.43.248.207) ``` If asked to do a `xfs_growfs` on `/var/lib/kubelet/...` it will fail with ``` could not find block size of device /var/lib/kubelet/plugins... ``` Only on the share-manager pod can we find the right mount path and filesystem type: ``` /dev/longhorn/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b on /export/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b type ext4 (rw,relatime) ``` We need the mount point to do resize through `ResizeFs.Resize`, which takes both the device path and the mount path: https://github.com/kubernetes/mount-utils/blob/e448c96afa03f6e2556117ed7ce7dee0462fa3ca/resizefs_linux.go#L46-L68 Specifically, that's because it knows that while resize2fs can operate on either one, xfs_growfs requires the mount path: ```go klog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) switch format { case "ext3", "ext4": return resizefs.extResize(devicePath) case "xfs": return resizefs.xfsResize(deviceMountPath) case "btrfs": return resizefs.btrfsResize(deviceMountPath) } return false, fmt.Errorf("ResizeFS.Resize - resize of format %s is not supported for device %s mounted at %s", format, devicePath, deviceMountPath)" ``` If we try to use `xfs_growfs` on the device, ``` longhorn-csi-plugin-pt7cj:/ # xfs_growfs /dev/longhorn/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b xfs_growfs: /dev/longhorn/pvc-8d1d3f46-76b6-484e-9a3a-93cc821ffd5b is not a mounted XFS filesystem ``` ## Design ### Implementation Overview The code in [longhorn-manager/csi/node-handler.go](https://github.com/longhorn/longhorn-manager/blob/235b1ae23f6ad4d26dc56063c95358145e937c09/csi/node_server.go#L732-L793) needs to be duplicated in the share-manager RPC server. That boils down to - construct the device path - resize the crypto device, if encrypted - resize the filesystem Crypto resize requires the passphrase, which is part of the `NodeExpandVolume` CSI request payload. But because the share-manager does the work, there is no need to send the passphrase over the wire to it. Share-manager already knows the crypto details from when its pod was created. It also knows the volume, too, so the new `FilesystemResize` method in its RPC server doesn't need any parameters at all. How does the request get sent to the share-manager? The longhorn-csi-plugin pod will make a gRPC call directly to share manager pod to run the filesystem resize command. For that, the csi-plugin needs to get a kubectl client to look up the pod for the associated share manager, and then also a share-manager RPC client. - What about multiple csi-plugin pods all forwarding to the one recipient? Is that a problem? Likely not. The resize should be idempotent. In fact, before calling `resizer.Resize`, the code checks `resizer.NeedsResize` and every instance after the first should just return. ### Test plan See steps to reproduce in the Github issue https://github.com/longhorn/longhorn/issues/9736. ### Upgrade strategy Any change to the share-manager pod requires a pod restart after upgrade to pick up the new image. That can be done at the user's convenience, but the new functionality won't be available until then. That should be documented in the release notes. If a FilesystemResize RPC is sent to an older share-manager, it will result in a "not implemented" error. The client should catch that, and append additional error text noting that it will be necessary to scale the workload down and then up again for the resize to take place. That will have the effect of upgrading the share-manager image as well, so any subsequent expansion of the volume can be done with the workload online. Only the first one is disruptive. ## Note [optional] *Additional notes.* ================================================ FILE: enhancements/20241203-v2-backing-image-support.md ================================================ # V2 Backing Image Support ## Summary This feature enables users to create and manage v2 backing images based on SPDK. With this feature, users can create a v2 volume with a v2 backing image to utilize the data stored in the backing image. ### Related Issues https://github.com/longhorn/longhorn/issues/6341 ## Motivation ### Goals #### V2 Backing Image Management - Users are able to `create` a v2 backing image with `raw` or `qcow2` image. - Each backing image copy is stored in a disk as a spdk lvol snapshot. - Users are able to `sync` the backing image copy from one disk to another. - Users are able `get` and monitor the information of the current backing image copy status. - Users are able to `delete` the backing image copy which is a lvol snapshot in the disk. #### With v2 Volume - Users are able to create a v2 volume with a v2 backing image copy. - The volume can be backed up and restored with the backing image. - The replicas of the volume can be rebuilt with the backing image. - The volume can be auto salvaged with the backing image. ### Non-goals - Fix the issue of inconsistent checksum led by skipping zero when dumping the data to the backing image lvol. - issue: https://github.com/longhorn/longhorn/issues/9876 - Allow users to backup a v2 backing image - https://github.com/longhorn/longhorn/issues/9992 - Allow users to export a v2 volume to a backing image - https://github.com/longhorn/longhorn/issues/9994 - Allow users to clone a v2 backing image. - https://github.com/longhorn/longhorn/issues/9996 ## Proposal ### User Stories #### V2 backing image management Users are able to create a v2 backing image through specifying a `dataEngine` field in the spec of the CRD. Longhorn will then automatically create the backing image copy on one of the disk with the source. Users can update the `diskFileSpecMap` in the backing image CRD spec to `sync` the backing image copy to other disks or to `delete` the backing image copy on other disks. Users can delete the backing image CR and all the backing image copies on every disks will be deleted. Like v1 backing image, Longhorn maintains the number `minNumberOfCopies` of copies in the cluster to reduce the possibility of the data loss. Like v1 backing image, Longhorn automatically cleans up the copies that are not used for a while to improve the space efficiency. #### With the v2 volume When the replicas require the backing image during the volume creation, the replica head lvol will be cloned from the backing image snapshot lvol. That backing image will become the first snapshot in the snapshot chain and the volume can read the data from it. This v2 volume should contain the same data as a v1 volume with a v1 backing image. This feature won't effect other volume's functionality. Volumes can still be rebuilt, backed up, restored and auto salvaged. Note that when backing up a v1 volume with a v1 backing image, Longhorn automatically backs up the backing image. However, for v2, the v2 backing image is not backed up, as this feature has not yet been implemented. Therefore, users must create the v2 backing image in the new cluster in advance if they wish to restore the v2 volume there. ### Implementation Overview #### CRD - Add `DataEngine` to the Spec and each disk spec and status of the backing image ``` type BackingImageDiskFileStatus struct { DataEngine DataEngineType `json:"dataEngine"` ... } type BackingImageDiskFileSpec struct { ... DataEngine DataEngineType `json:"dataEngine"` } type BackingImageSpec struct { DiskFileSpecMap map[string]*BackingImageDiskFileSpec `json:"diskFileSpecMap"` ... DataEngine DataEngineType `json:"dataEngine"` } type BackingImageStatus struct { DiskFileStatusMap map[string]*BackingImageDiskFileStatus `json:"diskFileStatusMap"` ... V2FirstCopyStatus BackingImageState `json:"v2FirstCopyStatus"` V2FirstCopyDisk string `json:"v2FirstCopyDisk"` } ``` - Add `BackingImage` to the instance manager CRD. ``` type InstanceManagerStatus struct { InstanceEngines map[string]InstanceProcess `json:"instanceEngines,omitempty"` InstanceReplicas map[string]InstanceProcess `json:"instanceReplicas,omitempty"` BackingImages map[string]BackingImageV2CopyInfo `json:"backingImages"` ... } type BackingImageV2CopyInfo struct { Name string `json:"name"` UUID string `json:"uuid"` DiskUUID string `json:"diskUUID"` Size int64 `json:"size"` Progress int `json:"progress"` State BackingImageState `json:"state"` CurrentChecksum string `json:"currentChecksum"` Message string `json:"message"` } ``` #### BackingImage Controller - We break the V2 backing image life cycle into 2 parts - Prepare first v2 backing image copy. - Manage v2 backing image copy in each disk periodically. 1. V2 Backing Image Preparation - Prepare first v1 backing image file - Same as the original flow - Utilize `backing-image-data-source` and `backing-image-manager` to prepare first v1 backing image copy with the source. - Prepare first v2 backing image copy - With field: `V2FirstCopyStatus` and `V2FirstCopyDisk` - Wait until there is a v1 backing image file and is transferred to `backing-image-manager` - The state transition of the `V2FirstCopyStatus` will be: `pending -> inProgress -> ready/failed/unknown` - Step1: Choose a v2 disk and create the backing image with the download url from the `backing-image-manager` - Step2: If the copy is `failed` or `unknown`, delete the copy and set the status to `pending` to restart the process. It will retry with a backoff. - Step3: If the copy is ready, clean up the v1 file disk. 2. Manage v2 backing image copy in each disk periodically - deleteInvalidV2Copy: - Delete the copy in the disk if the disk is removed from the spec. - If there is a ready copy - Delete the unknown copies if **there is no status and lvol** on the disk - Delete the failed copies - prepareV2Copy: - If there is a ready copy - Create a copy on the disk specified in the CR spec but does not have a status yet. - Only create one at a time. - syncV2StatusWithInstanceManager: - V2 backing image is managed in the spdk server in the instance manager - The status of the snapshot lvol will be monitored and stored in the CR status of the instance manager - We iterate all the instance manager to get the status of the backing image on each disk and update the backing image CR status. - If the copy has status before but fails to ge t the status from the instance manager this time, we mark it as `Unknown` Noted, if the node is rebooted, the in-memory status of the lvol in the spdk server will temporarily disappear. The status will become `Unknown`. However, the spdk server can pick up the backing image lvol and reconstruce the in-memory status. Thus, we only delete the `Unknown` copies when the lvol is also missing. #### Backing Image Manager - Data Source: Add a new parameter `DataEngine`. If it a is v2 backing image and the file format is qcow2, we convert it to raw when preparing the backing image file. So the checksum and the size can be consistent to the final v2 backing image lvol snapshot. - Backing Image Manager: Add a parameter to the download backing image endpoint. The SPDK Server will use this endpoint to download the backing image data and store it in an lvol to create a v2 backing image. In this case, the endpoint does not need to compress the data. #### Instance Manager Controller - Add a backing image monitor utilizing engine proxy client to get the backing image lvol snapshot status from the spdk server. - Add a set of spdk backing image APIs in engine proxy so controller can operate the backing image through the engine proxy. ``` SPDKBackingImageCreate(name, backingImageUUID, diskUUID, checksum, fromAddress, srcDiskUUID string, size uint64) (*imapi.BackingImage, error) SPDKBackingImageDelete(name, diskUUID string) error SPDKBackingImageGet(name, diskUUID string) (*imapi.BackingImage, error) SPDKBackingImageList() (map[string]longhorn.BackingImageV2CopyInfo, error) SPDKBackingImageWatch(ctx context.Context) (*imapi.BackingImageStream, error) ``` #### SPDK server - Add a set of backing image APIs ``` BackingImageCreate BackingImageDelete BackingImageGet BackingImageList BackingImageWatch BackingImageExpose BackingImageUnexpose ``` - Backing Image Create 1. Create a temp head `bi-${biName}-disk-${lvsUUID}-temp-head` lvol 2. Expose the temp head and create a target device 3. If `download` from the backing-image-manager URL - HTTP request the data and copy into the lvol 4. If `sync` from other spdk server lvol snapshot - Request the source to expose the backing image snapshot by `BackingImageExpose` - Connect the external snapshot and setup a source device - Copy the data from the source device to the target device - Request the source to unexpose the backing image. 5. Create a snapshot `bi-${biName}-disk-${lvsUUID}` from the temp head and stored the following info to the xattr - `backingImageUUID`. - `state`. So we know if this prepation is ready or failed. - `checksum`. - Verify - For records already existing, we check if the lvol exists. - If lvol is missing, we mark the status to `Failed`, so the controller can recreate the backing image copy on this disk. - For records missing, we reconstruct the record from the info stored in the xattr: `backingImageUUID`, `state` and `checksum`. - For replica creation - If it is created from a backing image, we clone the replica head from the backing image lvol snapshot and resize the head. - We put the backing image to the `r.ActiveChain[0]` - We add a new field `r.BackingImage` to stored the backing image lvol. - For replica reconstructing `ActiveChain` - It check the **parent of the ancestor** of the replica to see if it has a backing image. - It gets backing image lvol and put it to the `r.ActiveChain[0]` and set the `r.BackingImage` - For replica rebuilding - Since backing image name is `bi-${biName}-disk-${lvsUUID}` - When rebuilding the snapshot which is after the backing image, we need to rename the backing image name from the source replica from `bi-${biName}-disk-${srclvsUUID}` to `bi-${biName}-disk-${dstlvsUUID}` so the dst replica can find the backing image on its disk. --- ### Test plan 1. V2 Backing Image Creation - Create the backing images from the following YAML. - Create 3 volumes with the following 3 backing images. - Attach the volumes to the node. - They should have the same checksum. ``` # v1 apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot-v1-raw namespace: longhorn-system spec: dataEngine: v1 minNumberOfCopies: 1 sourceType: download sourceParameters: url: https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.raw --- # raw apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot-v2-raw namespace: longhorn-system spec: dataEngine: v2 minNumberOfCopies: 1 sourceType: download sourceParameters: url: https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.raw --- # qcow2 apiVersion: longhorn.io/v1beta2 kind: BackingImage metadata: name: parrot-v2-qcow2 namespace: longhorn-system spec: dataEngine: v2 minNumberOfCopies: 1 sourceType: download sourceParameters: url: https://longhorn-backing-image.s3-us-west-1.amazonaws.com/parrot.qcow2 ``` 2. V2 Backing Image Creation - Create the v2 backing image with all kind of the sources - Upload - Download - Export - Clone - Backup a v1 BackingImage and restore it to v2 3. Regression1 - Replica Rebuilding - Create a v2 volume with the v2 backing image - Delete one of the replica - The volume should become healthy again after replica rebuilding 4. Regression1 - Auto Salveage - Create a v2 volume with the v2 backing image - Delete all of the instance-manager - The volume should become healthy again after all instance manager come back. 5. Regression1 - Backup/Restore - Create a v2 volume with the v2 backing image - Attach it and write some data and get the checksum - Backup the volume - Restore the volume with the same v2 backing image - Attach it and get the checksum to see if the data is correct. ### Upgrade strategy - All backing images before v1.8.0 should be `DataEngine=v1` - All disks' spec and status of backing images before v1.8.0 should be `DataEngine=v1` ## Note [optional] Additional notes. ================================================ FILE: enhancements/20241206-v2-volume-live-migration.md ================================================ # V2 Volume Live Migration ## Summary Implement live migration feature for volume with v2 data engine ### Related Issues https://github.com/longhorn/longhorn/issues/6361 ## Motivation ### Goals 1. Implement live migration feature for volume with v2 data engine similar to live migration feature for volume with v1 data engine. 2. Make sure we can handle disaster test cases during live migration ## Proposal ### User Stories Live migration for v2 is needed for Harvester VM live migration feature ### User Experience In Detail From the user's perspective, the experience will be very similar to live migration feature in v1 volume First, user create SC with `migratable` set to `true` and `dataEngine` set to `v2` ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-v2-migratable provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "1" staleReplicaTimeout: "2880" # 48 hours in minutes dataEngine: "v2" migratable: "true" ``` Then user create PVC from the SC with `accessModes` as `ReadWriteMany` and `volumeMode` as `Block` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-block-vol namespace: default spec: accessModes: - ReadWriteMany volumeMode: Block storageClassName: longhorn-v2-migratable resources: requests: storage: 2Gi ``` Then user deploy a pod using the PVC ```yaml apiVersion: v1 kind: Pod metadata: name: pod-1 namespace: default spec: nodeName: node-1 containers: - name: block-volume-test image: ubuntu:xenial imagePullPolicy: IfNotPresent command: ["sleep", "36000"] volumeDevices: - devicePath: /dev/longhorn/testblk name: block-vol volumes: - name: block-vol persistentVolumeClaim: claimName: longhorn-block-vol ``` If the user want to migrate the pod to node-2, they can deploy an identical pod on node-2 using the same PVC ```yaml apiVersion: v1 kind: Pod metadata: name: pod-2 namespace: default spec: nodeName: node-2 containers: - name: block-volume-test image: ubuntu:xenial imagePullPolicy: IfNotPresent command: ["sleep", "36000"] volumeDevices: - devicePath: /dev/longhorn/testblk name: block-vol volumes: - name: block-vol persistentVolumeClaim: claimName: longhorn-block-vol ``` Longhorn will expose the volume to both node-1 and node-2. 1. User then can do operation like flush/pausing IO on pod-1 and delete pod-1. This would mean migration confirmation. Longhorn would then detach the volume from node-1. The volume become solely attached to node-2. 2. User can also delete the pod-2. This would mean migration rollback Longhorn would then detach the volume from node-2. The volume become solely attached to node-1. ### API changes Adding a new field `MigratingEngineName` to the replica CR ```go // ReplicaSpec defines the desired state of the Longhorn replica type ReplicaSpec struct { InstanceSpec `json:""` // +optional EngineName string `json:"engineName"` // +optional // MigratingEngineName is indicating the migrating engine which current connected to this replica. This is only // used for live migration of v2 data engine MigratingEngineName string `json:"migratingEngineName"` ``` This field is used to indicating which engine the replica is migrating to. This will be explained in more details in the section [Implementation Overview](#implementation-overview). ## Design ### Implementation Overview In v1 volume live migration, the flow was: 1. Assume that volume has engine e1 with replica r1 and r2 1. Create a new inactive engine e2 on node2 1. Create inactive replicas r1' and r2' that match spec of r1 and r2. r1' is accessing same replica folder as r1. r2' is accessing same replica folder as r2 1. Start replica r1' and r2' 1. Start engine e2 with replica r1' and r2' 1. Now migration is ready. User can either do migration confirmation or migration rollback 1. If user does migration confirmation: 1. Longhorn delete e1 1. Active e2 1. Active r1' and r2' 1. Deactivate r1 and r2 1. If user does migration rollback: 1. Longhorn delete e2 1. Delete r1' and r2' The reason, we have to create new inactive replicas r1' and r2' is because v1 replica is designed to be connected to a single engine at time. However, it seems that v2 replica can be connected by multiple engine at a time. More over, creating and matching new inactive replicas in v1 live migration seems a bit complicated especially when it comes to handle migration revert. Therefore, I decided to not creating new inactive replicas in v2 live migration. The flow would be like this: ![v2 volume live migration flow](./assets/images/v2-volume-live-migration/v2-volume-live-migration-flow.png) Detailed state transitions: ![v2 volume live migration state transition](./assets/images/v2-volume-live-migration/v2_live_migration_state_transition.png) ### Test plan Test cases: 1. Migration confirmation 1. Migration rollback 1. Engine crash during migration 1. Original engine crash 1. Migrating engine crash 1. Replica crash during live migration: 1. Some replicas crash 1. All replicas crash: failed 1. Run all e2e migration test cases More details are at https://github.com/longhorn/longhorn/issues/6361#issuecomment-2537547340 ### Upgrade strategy No upgrade strategy is needed ================================================ FILE: enhancements/20250107-v2-volume-encryption.md ================================================ # V2 Volume Encryption ## Summary This enhancement adds support for user configured (storage class, secrets) encrypted v2 volumes. ### Related Issues - https://github.com/longhorn/longhorn/issues/7355 ## Motivation ### Goals - User is able to create and use an encrypted v2 volume with cipher customization options only via CSI. ## Proposal There are two candidates for the v2 volume encryption. 1. The `dm_crypt` kernel module (Linux kernel device-mapper crypto target) as the v1 volume encryption. Device-Mapper’s “crypt” target provides transparent encryption of block devices using the kernel crypto API. 2. The SPDK [Crypto Virtual Bdev Module](https://spdk.io/doc/bdev.html). The SPDK has the crypto virtual bdev module to provide at rest data encryption for any underlying bdev. The module relies on the `SPDK Accel Framework` to provide all cryptographic functionality. Benchmark information: - Equinix metal machine: m3.large.x86 - CPU: 1 x AMD EPYC 7502P and 32 cores @ 2.50GHz - Memory: 256 GB - Test Disks: 2 x 3.8 TB NVMe - NICs: 2 x 25 Gbps - Volume size: 50 GiB - Replicas for the volume: 2 | [SPDK Crypto Virtual Module](https://spdk.io/doc/bdev.html) / CBC | Sequential | Random | | --------------------------------: | :-------------------: | :-------------------: | | IOPS (Read/Write) | 240,125 / 235,990 | 239,201 / 236,695 | | Bandwidth in KiB/sec (Read/Write) | 2,022,358 / 1,478,101 | 1,990,292 / 1,420,072 | | Latency in ns (Read/Write) | 165,033 / 161,666 | 164,378 / 162,669 | | `dm_crypt` kernel module / CBC | Sequential | Random | | --------------------------------: | :-------------------: | :-------------------: | | IOPS (Read/Write) | 332,107 / 226,378 | 332,758 / 61,755 | | Bandwidth in KiB/sec (Read/Write) | 4,866,733 / 1,878,130 | 4,838,664 / 1,947,787 | | Latency in ns (Read/Write) | 169,440 / 270,217 | 169,967 / 271,265 | The performance discrepancy between an encrypted SPDK device using the `dm_crypt` kernel module and SPDK's native crypto virtual bdev nodule. - Kernel space and user space: - The `dm_crypt` module operates entirely in the Linux kernel space - For the crypto virtual bdev nodule, the processing might happen in user space, which can introduce additional memory copying and context switching when compared to kernel space encryption. - Hardware acceleration: - The `dm_crypt` module leverages hardware encryption acceleration features provided by the CPU or dedicated hardware that Linux kernel can use. - While SPDK supports hardware acceleration, it requires explicit configuration to utilize a hardware-based cryptographic device (e.g., `Intel QuickAssist` or DPDK-supported accelerators) Based on the benchmark comparison between the [SPDK Crypto Virtual Module](https://spdk.io/doc/bdev.html) and the `dm_crypt` kernel module, v2 volume encryption will utilize the `dm_crypt` kernel module for encrypting the v2 volumes. ### User Stories All regular longhorn operations should also be supported for encrypted v2 volumes, therefore the only user story that is mentioned is how to create and use an encrypted v2 volume. #### Create And Use An Encrypted V2 Volume - Create a storage class with (encrypted=true and dataEngine=v2) and either a global secret or a per volume secret. - Create the secret for that volume in the configured namespace with customization options of the cipher for instance `cipher`, `key-size` and `hash`. - Create a PVC that references the created storage class. - A v2 volume will be created then encrypted during first use. - Afterwards a regular filesystem that lives on top of the encrypted volume or the raw block volume will be exposed to the pod. ### User Experience In Detail Creation and usage of an encrypted v2 volume requires: - The encrypted v2 volumes only support CSI as encrypted v1 volumes. - The storage class needs to specify `encrypted: "true"` and `dataEngine: "v2"` as part of its parameters. - Secrets need to be created and reference for the CSI operations need to be setup. #### Create Storage Class With A Global Secret The storage class uses a global secret named `longhorn-crypto` in the `longhorn-system` namespace. ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-global provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes ... encrypted: "true" dataEngine: "v2" csi.storage.k8s.io/provisioner-secret-name: "longhorn-crypto" csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-expand-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-expand-secret-namespace: "longhorn-system" ``` ## Design ### Implementation Overview Host requires `dm_crypt` kernel module as well as `cryptsetup` installed as the v1 volume requirement. - Utilize host `dm_crypt` kernel module for device encryption. - Utilize host installed `cryptsetup` for configuration of the crypto device. - CSI flows `NodeStageVolume`, `NodeUnstageVolume`, `NodePublishVolume` and `NodeExpandVolume` will operate as the v1 volume does. - During CSI `NodeStageVolume` encrypts the regular longhorn device (/dev/longhorn/[volume-name]). - It exposes a crypto mapped device (/dev/mapper/[volume-name]-encrypted). Since the v2 volume will use the device mapper to control I/O and the `dm` device name will match the volume name, we add the suffix `-encrypted` to the encrypted `dm` device in the `/dev/mapper/` directory to resolve the naming conflict, as illustrated in the stack graph below. ![v2 encrypted volume stack](./image/v2-encrypted-volume-stack.png) ### Test plan #### Create An Encrypted V2 Volume And Verify Encryption of Volume - Create the secret for that volume in the configured namespace - Create a storage class with (encrypted=true and dataEngine=v2) and either a global secret. - Create a PVC that references the created storage class - Create a pod that uses that PVC for a volume mount - Wait for pod up and healthy - Write the known test pattern into the file system - Verify absence of the known test pattern after directly reading block device content `/dev/longhorn/[volume-name]` #### Create An Encrypted V2 Volume With The Customized Cipher - Create the secret with customized options of the cipher for that volume in the configured namespace - Create a storage class with (encrypted=true and dataEngine=v2) and either a global secret. - Create a PVC that references the created storage class - Create a pod that uses that PVC for a volume mount - Wait for pod up and healthy - Check if the customized cipher are correct ================================================ FILE: enhancements/20250313-ublk-frontend-for-v2-engine.md ================================================ # UBLK Frontend For V2 Engine ## Summary Longhorn v2 data engine current only support NVME-TCP frontend. We would like to support UBLK frontend for v2 data engine. In high performance environment, UBLK frontend may have better performance compares to NVME-TCP. Ref https://github.com/longhorn/longhorn/wiki/Longhorn-Performance-Investigation ### Related Issues https://github.com/longhorn/longhorn/issues/9456 ## Motivation ### Goals Support UBLK frontend for v2 data engine ### Non-goals In this ticket, supporting live engine upgrade is not a goal. It will be handled in different ticket https://github.com/longhorn/longhorn/issues/9104 ## Proposal ### User Stories In high performance env, UBLK frontend may have better performance compares to NVME-TCP. Ref https://github.com/longhorn/longhorn/wiki/Longhorn-Performance-Investigation ### User Experience In Detail UBLK framework added in the Linux 6.0 kernel. User need to have kernel version >= 6.0 to use this feature. Require kernel module `ublk_drv` In order to use UBLK frontend for v2 data engine, user just need to create a v2 volume with `volume.Spec.Frontend: ublk`. For example: ```yaml apiVersion: v1 items: - apiVersion: longhorn.io/v1beta2 kind: Volume metadata: name: testvol namespace: longhorn-system spec: Standby: false accessMode: rwo backendStoreDriver: "" backingImage: "" backupCompressionMethod: lz4 backupTargetName: default dataEngine: v2 dataLocality: disabled dataSource: "" disableFrontend: false diskSelector: [] encrypted: false engineImage: "" freezeFilesystemForSnapshot: ignored fromBackup: "" frontend: ublk image: longhornio/longhorn-instance-manager:master-head lastAttachedBy: "" migratable: false migrationNodeID: "" nodeID: "" nodeSelector: [] numberOfReplicas: 1 replicaAutoBalance: ignored replicaDiskSoftAntiAffinity: ignored replicaSoftAntiAffinity: ignored replicaZoneSoftAntiAffinity: ignored restoreVolumeRecurringJob: ignored revisionCounterDisabled: true size: "21474836480" snapshotDataIntegrity: disabled snapshotMaxCount: 250 snapshotMaxSize: "0" staleReplicaTimeout: 20 unmapMarkSnapChainRemoved: disabled ``` ### API changes We will add a new field to the engine CR's status: `UblkID` indicating the current ID of the ublk device on node corresponding to this engine. For example, ```yaml apiVersion: v1 items: - apiVersion: longhorn.io/v1beta2 kind: Engine metadata: creationTimestamp: "2025-02-19T23:49:20Z" finalizers: - longhorn.io generation: 19 labels: longhornnode: "" longhornvolume: testvol name: testvol-e-0 namespace: longhorn-system ownerReferences: - apiVersion: longhorn.io/v1beta2 kind: Volume name: testvol uid: 5097129b-cbd8-496d-92c9-107ebd7c4e98 resourceVersion: "6620798" uid: a3516605-403b-4a9e-9e8c-67f1d040564a spec: active: true backendStoreDriver: "" backupVolume: "" dataEngine: v2 desireState: stopped disableFrontend: false engineImage: "" frontend: blockdev image: longhornio/longhorn-instance-manager:master-head logRequested: false nodeID: "" replicaAddressMap: testvol-r-8c244275: 10.42.7.177:20001 requestedBackupRestore: "" requestedDataSource: "" revisionCounterDisabled: true salvageRequested: false snapshotMaxCount: 250 snapshotMaxSize: "0" unmapMarkSnapChainRemovedEnabled: false upgradedReplicaAddressMap: {} volumeName: testvol volumeSize: "21474836480" status: backupStatus: null cloneStatus: null conditions: - lastProbeTime: "" lastTransitionTime: "2025-02-19T23:49:20Z" message: "" reason: "" status: "True" type: InstanceCreation - lastProbeTime: "" lastTransitionTime: "2025-02-19T23:49:20Z" message: "" reason: "" status: "False" type: FilesystemReadOnly currentImage: "" currentReplicaAddressMap: testvol-r-8c244275: 10.42.7.177:20001 currentSize: "21474836480" currentState: stopped endpoint: "" instanceManagerName: "" ip: "" isExpanding: false lastExpansionError: "" lastExpansionFailedAt: "" lastRestoredBackup: "" logFetched: false ownerID: phan-v715-pool2-wn2t9-p8cjh port: 0 ublkID: 1 purgeStatus: null rebuildStatus: null replicaModeMap: null restoreStatus: null salvageExecuted: false snapshotMaxCount: 0 snapshotMaxSize: "0" snapshots: volume-head: children: {} created: "2025-02-19T23:49:34Z" labels: {} name: volume-head parent: "" removed: false size: "0" usercreated: true snapshotsError: "" started: false storageIP: "" unmapMarkSnapChainRemovedEnabled: false ``` Longhorn UI will also be changed to allow the user to create the new v2 volume with UBLK frontend ## Design This is an overview of the NVME-TCP frontend stack VS the new UBLK frontend stack ![Example Image](./assets/images/ublk-frontend-for-v2-engine/nvme-tcp-frontend-vs-ublk-frontend.png) ### Implementation Overview #### Refactor the Engine struct in `longhorn-spdk-engine/pkg/spdk/engine.go` Refactor the Engine struct in `longhorn-spdk-engine/pkg/spdk/engine.go` to separate the information about NVME-TCP frontend and UBLK frontend. This is the old struct: ```go type Engine struct { sync.RWMutex Name string VolumeName string SpecSize uint64 ActualSize uint64 IP string Port int32 // Port that initiator is connecting to TargetIP string TargetPort int32 // Port of the target that is used for letting initiator connect to StandbyTargetPort int32 Frontend string Endpoint string Nqn string Nguid string ctrlrLossTimeout int fastIOFailTimeoutSec int ReplicaStatusMap map[string]*EngineReplicaStatus initiator *nvme.Initiator dmDeviceIsBusy bool State types.InstanceState ErrorMsg string Head *api.Lvol SnapshotMap map[string]*api.Lvol IsRestoring bool RestoringSnapshotName string // UpdateCh should not be protected by the engine lock UpdateCh chan interface{} log *safelog.SafeLogger } ``` And this is the new struct ```go type Engine struct { sync.RWMutex Name string VolumeName string SpecSize uint64 ActualSize uint64 Frontend string Endpoint string ctrlrLossTimeout int fastIOFailTimeoutSec int ReplicaStatusMap map[string]*EngineReplicaStatus NvmeTcpFrontend *NvmeTcpFrontend UblkFrontend *UblkFrontend initiator *nvme.Initiator dmDeviceIsBusy bool State types.InstanceState ErrorMsg string Head *api.Lvol SnapshotMap map[string]*api.Lvol IsRestoring bool RestoringSnapshotName string // UpdateCh should not be protected by the engine lock UpdateCh chan interface{} log *safelog.SafeLogger } type NvmeTcpFrontend struct { IP string Port int32 // Port that initiator is connecting to TargetIP string TargetPort int32 // Port of the target that is used for letting initiator connect to StandbyTargetPort int32 Nqn string Nguid string } type UblkFrontend struct { UblkID int } ``` Now the engine with NVME-TCP frontend will have the field `NvmeTcpFrontend != nil` which contains the information related to NVME-TCP only. And the engine with the UBLK frontend will have the field `UblkFrontend != nil` which contains the information related to UBLK only #### Refactor the Initiator struct in `go-spdk-helper/pkg/nvme/initiator.go` Similarly, refactor Initiator struct in `go-spdk-helper/pkg/nvme/initiator.go` to separate the information about NVME-TCP frontend and UBLK frontend. This is the old Initiator struct ```go type Initiator struct { Name string SubsystemNQN string UUID string TransportAddress string TransportServiceID string Endpoint string ControllerName string NamespaceName string dev *util.LonghornBlockDevice isUp bool hostProc string executor *commonns.Executor logger logrus.FieldLogger } ``` And this is the new Initiator struct: ```go type Initiator struct { Name string Endpoint string dev *util.LonghornBlockDevice isUp bool NvmeTcpInfo *NvmeTcpInfo UblkInfo *UblkInfo hostProc string executor *commonns.Executor logger logrus.FieldLogger } type NvmeTcpInfo struct { SubsystemNQN string UUID string TransportAddress string TransportServiceID string ControllerName string NamespaceName string } type UblkInfo struct { BdevName string UblkID int } ``` #### Build more SPDK API in `go-spdk-helper/pkg/spdk/client/basic.go` These SPDK APIs are: * `UblkCreateTarget`: Start to create ublk threads and initialize ublk target. It will return an error if user calls this RPC twice without ublk_destroy_target in between. It will use current cpumask in SPDK when user does not specify cpumask option. * `UblkDestroyTarget`: Release all UBLK devices and destroy ublk target. * `UblkGetDisks`: Display full or specified ublk device list * `UblkStartDisk`: Start to export one SPDK bdev as a UBLK device * `UblkRecoverDisk`: Recover original UBLK device with ID and block device * `UblkStopDisk`: Delete a UBLK device #### The flow when creating a new v2 engine with new UBLK frontend 1. User creates a volume with `Spec.Frontend: ublk` 1. Longhorn manager creates engine CR with `Spec.Frontend: ublk` 1. Longhorn manager also creates replica CRs 1. Longhorn manager starts replica and expose them as NVME-TCP target as usually 1. Longhorn manager send gRPC call to instance manager to create engine with UBLK frontend 1. Instance manager connect to the replicas 1. Instance manager creates a raid1 bdev from these replicas' NVME bdevs 1. Instance manager create ublk frontend by calling `handleUblkFrontend` 1. Instance manager starts ublk target if it has not been started by calling `UblkCreateTarget` 1. UBLK initiator cleanups the previous ublk device associated with the raid1 bdev by calling `UblkGetDisks` and `UblkStopDisk` 1. UBLK initiator finds an available ublk id by calling `UblkGetDisks` 1. UBLK initiator creates new ublk device for the raid bdev by calling `UblkStartDisk` 1. UBLK initiator set `UblkInfo.UblkID` to the available ID 1. UBLK initiator links that device to dm mapper 1. UBLK initiator duplicates dm device to `e.Endpoint` #### The flow when deleting a new v2 engine with new UBLK frontend 1. Longhorn manager send gRPC call to instance manager to delete engine with UBLK frontend 1. UBLK initiator removes dm mapper device 1. UBLK initiator removes `e.Endpoint` 1. UBLK initiator delete ublk device associated with the raid1 bdev by calling `UblkStopDisk` with the recorded `UblkInfo.UblkID` info 2. Instance manager disconnects from the replicas 1. Longhorn manager delete replica and stops expose them as NVME-TCP target ### Test plan https://github.com/longhorn/longhorn/issues/9456#issuecomment-2811508087 ### Upgrade strategy No upgrade strategy is needed ================================================ FILE: enhancements/20250331-orphaned-runtime-cleanup.md ================================================ # Orphaned Engine And Replica Runtime Instance Cleanup ## Summary Orphaned runtime instance cleanup identifies unmanaged engine and replica runtime instances on the nodes, and provides a list of the orphan instances on each node. Longhorn by default keeps the instance. Also, it provides a configurable way to clean up such resources automatically. ### Related Issues [https://github.com/longhorn/longhorn/issues/6764](https://github.com/longhorn/longhorn/issues/6764) ## Motivation ### Goals - Identify the orphaned engine and replica processes - The scanning process should not stick to the reconciliation of the controller - Provide the user a way to select and trigger the deletion of the orphaned runtime instance - Support the global auto-deletion of orphaned runtime - Compatible with existing cluster, and is able to clean up the existing orphaned runtime in the acceptable previous versions. ### Non-goals - Clean up orphaned data stores on the nodes - Support the per-node auto-deletion of orphaned runtime - Support the auto-deletion of orphaned runtime exceeding the TTL - Prevent the process deletion race between CRs (see the design note below) ## Proposal 1. Introduce new types of existing CRD `orphan` that represents and tracks the orphaned runtime instances. This new type of orphan CR is reconciled by existing orphan controller. The orphan controller deletes the runtime instance if receives a deletion request. The orphan CR reflects the existence of the runtime instance. 2. The monitor on each instance manager controller keeps watching the runtime instances managed by instance manager, compares them with the scheduled engine and replica, and then finds the orphaned runtime instances. Once a runtime instance is considered to be orphaned, create a corresponding orphan CR, and keep update the instance state on this orphan CR. The monitor will delete the orphan CR when an instance disappears from the node, or, the corresponding engine/replica is scheduled back. 3. The node controller keeps tracking the node status. When a node is evicted or disconnects from the cluster, node controller initiates the deletion on orphan CRs belongs to this node. ### User Stories When a network outage occurs to some Longhorn node, it may contain multiple engine or replica runtime instances not tracked by the Longhorn System. The corresponding engine and replica CRs may be removed or relocated to another node during the outage. When the node comes back, the corresponding runtime resources are no longer tracked by the Longhorn system. These runtime resources, including the processes, are called orphaned. Orphaned runtime resources continue to consume CPU and memory. Users have no way to clean up such resources except forcibly restarting the entire instance manager pod on the node, which will unnecessarily increase the chance of a system outage. After the enhancement, Longhorn automatically finds out the orphaned runtime instances on Longhorn nodes. Users can visualize and manage the orphaned replica runtime instances via Longhorn GUI or command line tools. Additionally, Longhorn can delete the orphaned runtime resources automatically if users enable the global auto-deletion option. ### User Experience In Detail - Via Longhorn GUI - Users can check instance status and see if Longhorn already identifies orphaned engines/replicas. - Users can choose the items in the orphaned replica directory list and clean up them. - Users can enable the global auto-deletion on setting page. By default, the auto-deletion is disabled. - Via `kubectl` - Users can list the orphaned runtime instances directories by `kubectl -n longhorn-system get orphans`. - Users can delete the orphaned runtime instances directories by `kubectl -n longhorn-system delete orphan `. - Users can enable or disable the global auto-deletion by `kubectl -n longhorn-system edit settings orphan-engine-instance-auto-deletion` - Users can enable or disable the global auto-deletion by `kubectl -n longhorn-system edit settings orphan-replica-instance-auto-deletion` ## Design ### Implementation Overview - Orphan CRs track the liveness of corresponding orphaned runtime instance - Orphan CRs are created by instance manager monitor - Orphan CRs are deleted by - Longhorn node controller: when node is deleted or evicted - Instance manager monitor: - When runtime instance disappear, delete the corresponding orphan runtime instance CR. - When auto deletion is enabled, delete all orphan runtime instance CRs in this instance manager. - When runtime instance is rescheduled back to the instance manager, delete the corresponding orphan runtime instance CR since it is no longer orphaned. **Settings** - Add setting `orphan-resource-auto-deletion`. - This is a string of semicolon-separated list. Possible items: - `replica-data` to enable auto deletion on replica data store. This replaces the old setting `orphan-auto-deletion`. - `instance` to enable auto deletion on engine and replica runtime instance. - Default value is empty, which means to disable auto deletion for all kinds of orphaned resources. - While upgrade Longhorn to v1.9.0, default set to `replica-data` if old setting `orphan-auto-deletion` is enabled. - Delete `orphan-auto-deletion` **Instance manager controller** - Start the instance monitor during initialization. - Reconciles the settings events and orphan events. - When auto deletion enabled, delete the orphan runtime instance CRs in this instance manager. - When the instance manager is no longer running, delete the orphan runtime instance CRs in this instance manager. - Delete the exist orphan runtime instance CRs if the runtime instance disappeared from instance manager. - Delete the exist orphan runtime instance CRs if the corresponding engine/replica is scheduled back to this instance manager. **Instance manager monitor** - Establishes a GRPC stream with instance manager to watch the status of runtime instances. - Receives engine instance status update events from GRPC stream - Update runtime instance status on the list in instance manager CR's `status.instanceEngines` and `status.instanceReplicas` - Compare the known engine/replica runtime instances with corresponding engine/replica CRs: - If there's no such CR, this instance is considered to be orphaned. - If there's a corresponding CR: - If the `status.currentState` is different from `spec.desireState`, ignore it because the state and ownership may change. - If the corresponding CR is in running state: - If the owner ID is not node ID, ignore it. - The instance is considered to be orphaned if and only if the instance manager is not the current instance manager. - If the corresponding CR is in stopped state: - The instance is considered to be orphaned if and only if the instance manager is not the current instance manager. - To other corresponding CR states, ignore it because of unstable state. - Compare the orphaned engine/replica runtime instances with the exist orphan CRs - Create an orphan CR for each orphaned instance if missing. ``` ┌────────────────────┐ │ │ │ an engine/replica │ │ instance listed by │ │ instance manager │ │ │ └──────────┬─────────┘ │ ┌─────────▼─────────┐ │ │ │ compare the │ │ corresponding │ │ engine/replica CR │ │ │ └─────────┬─────────┘ │ ┌─────▼─────┐ │ │ not exist │ CR exist? ├──────────────────────────────┐ │ │ │ └─────┬─────┘ │ │ exist │ no, CR state ┌───────────────▼─────────────────────────┐ │ will change │ │ │ ┌─────────┤ status.currentState == spec.desireState │ │ │ │ │ │ │ └───────────────┬─────────────────────────┘ │ │ │ yes, stable state │ │ ┌───────▼────────┐ │ │ │ │ │ │ │ currentState ? │ │ │ │ │ │ │ └───────┬────────┘ │ │ │ │ │ ┌──────────────┼──────────────┐ │ │ │ │ │ │ │ ┌─────▼────┐ ┌────▼────┐ ┌────▼────┐ │ │ │ other │ │ │ │ │ │ │ │(starting,│ │ running │ │ stopped │ │ │ │ stopping,│ │ │ │ │ │ │ │ error )│ └────┬────┘ └────┬────┘ │ │ └─┬────────┘ │ │ │ │ │ │ │ │ ┌───▼──────▼──┐ ┌───────▼────────┐ │ │ │ │ │ │ │ │ │ no decision │ no │ CR owner == │ │ │ │ ◄───────┤ desired node ? │ │ │ └──────▲──────┘ │ │ │ │ │ └───────┬────────┘ │ │ │ yes │ │ │ │ ┌─────▼──────────────▼─────┐ ┌──────────▼───────────┐ │ │ │ │ │ │ │ CR IM == current IM? │ │ instance is orphaned │ │ │ │ │ │ │ └─────┬──────────────┬─────┘ └──────────▲───────────┘ └──────────────────────┘ └─────────────────────┘ CR in this instance manager CR in other instance manager ``` **Orphan CR** - The orphan CR name is calculated from engine/replica ID. - To engine instance: `orphan-${checksum}` - To replica instance: `orphan-${checksum}` - `$checksum = sha256("${engine_replica_name}-${instance_manager_id}-${data_engine_type}")` - labels: - `longhorn.io/component`: `orphan` - `longhorn.io/managed-by`: `longhorn-manager` - `longhorn.io/orphan-type`: `engine-instance` or `replica-instance` - `longhornnode`: node ID - `longhorninstancemanager`: instance manager ID - `longhornengine`: the instance name. Set only when it is an engine instance - `longhornreplica`: the instance name. Set only when it is a replica instance - The `OrphanType` is `engine-instance` or `replica-instance`. - Record the instance name in `Parameters["InstanceName"]`. - Record the instance manager in `Parameters["InstanceManager"]`. - Record the data engine type in `DataEngine`. - Record the runtime instance's state in `status.conditions[].Reason` with type `"InstanceState"` - If instance state is `terminated`, the condition's status will be `False`, indicates that the orphan CR is no longer needed to track this instance. - To other instance states, the condition's status will be `True`. The instance is present in instance manager. **Orphan controller** Reconciles the orphan events. - If the `deletionTimestamp` is non-zero, and the current controller is responsible for this orphan CR, proceed with deletion request. - If the orphan's node ID is the current controller ID: - If the instance state is deleted, remove the finalizer to complete deletion directly. - Otherwise, check with the corresponding engine or replica CR before deleting the runtime instance. - If the engine / replica CR not exists, this runtime instance is deletable. - If the engine / replica CR exists, and the instance manager ID is not the one in orphan CR, then the runtime instance is deletable. - Once the runtime instance is deletable, create an instance manager client from the instance manager CR, and delete the runtime instance. The finalizer will be removed in the future when the instance state got into deleted, or the runtime instance became not deletable. - If the runtime instance is not deletable, remove the finalizer to delete the orphan CR. - Otherwise, the orphan runtime instance lives on another node, but the controller loses its ownership because of disconnection. Remove the finalizer to complete deletion. **Longhorn node controller** Reconcile the Longhorn node events. - Delete orphan instance CRs on a deleted or evicted node **longhorn-ui** - Allow users to list the orphans on the node page by sending `OrphanList` call to the backend. - Allow users to select the orphans to be deleted. The frontend needs to send `OrphanDelete` call to the backend. ### Test Plan **Integration tests** - `orphan` CRs will be created correctly while the process instance created unexpectedly. And they can be cleaned up without touching the data store. - `orphan` CRs will be created correctly to a re-connected node. And they can be cleaned up without touching the data store. - `orphan` CRs will be removed when an engine / replica is relocated back to the node. - `orphan` CRs will be removed when the node is evicted or down. - Auto-deletion setting. ## Note[optional] The orphan CR and the orphan controller was designed for orphaned data store in LEP ["Orphaned Replica Directory Cleanup"](https://github.com/longhorn/longhorn/blob/master/enhancements/20220324-orphaned-data-cleanup.md). There's very little chance of deleting the instance process accidentally due to the race condition when an engine/replica is rescheduled back to the node. In this situation, after orphan controller sending delete request to instance manager, the engine/replica controller can still recover the instance process. As a future work, consider a breaking change to add some tags on instance processes to sync the status between the orphan CR and the corresponding engine/replica CR. ================================================ FILE: enhancements/20250407-volume-offline-rebuilding.md ================================================ # Volume Offline Rebuilding ## Summary This enhancement adds support for offline replica rebuild functionality for Longhorn volumes. It will allow rebuilding replicas while the volume is detached, enhancing the volume availability and reliability. ### Related Issues - https://github.com/longhorn/longhorn/issues/8443 ## Motivation ### Goals - Support enabling and disabling volume offline rebuilding by enabling the global setting `offline-replica-rebuilding`. - Support enabling and disabling volume offline rebuilding by the per-volume by the Longhorn UI, API or `kubectl` command. ### Non-goals - Support offline replica rebuilding for faulted volumes. ## Proposal ### API changes Introduce new volume Action APIs `offlineReplicaRebuilding` and a new field `Volume.Spec.OfflineRebuilding`: | API | Input | Output | Comments | HTTP Endpoint | | --- | --- | --- | --- | --- | | Update | N/A | err error | Enable/Disable volume offline rebuilding | **POST** `/v1/volumes/{VolumeName}?action=offlineReplicaRebuilding` | ```golang type UpdateOfflineRebuildInput struct { OfflineRebuilding string `json:"offlineRebuilding"` } type VolumeOfflineRebuilding string const ( VolumeOfflineRebuildEnabled = VolumeOfflineRebuilding("enabled") VolumeOfflineRebuildDisabled = VolumeOfflineRebuilding("disabled") VolumeOfflineRebuildIgnored = VolumeOfflineRebuilding("ignored") ) type VolumeSpec struct { ... // The backup target name that the volume will be backed up to or is synced. // +optional BackupTargetName string `json:"backupTargetName"` // The flag that enable the offline replica rebuilding when volume is detached. // +optional OfflineRebuilding VolumeOfflineRebuilding `json:"offlineRebuilding"` } ``` ### User Stories - Users want to automatically rebuild replicas while a degraded volume is detached to ensure maintain data redundancy. - Users want to start a workload with the volume when the volume is in offline replica rebuild process. - The offline rebuilding process will still work after the cluster goes down and comes back up. - A worker node is down during the offline rebuilding process. ### User Experience In Detail | `Volume.Spec.OfflineRebuilding` \ setting `offline-replica-rebuilding` | `true` | `false` | | :---: | :---: | :---: | | `ignored` | ✓ | X | | `enabled` | ✓ | ✓ | | `disabled` | X | X | - ✓: The offline rebuilding is enabled. - X: The offline rebuilding is disabled. #### Manually Enable An Individual Volume Offline Rebuilding When users want to enable the offline rebuilding of a detached volume: 1. After enabling the per-volume offline rebuilding, if the volume is degraded, it will start the rebuilding process. 2. When all replicas (`Spec.NumberOfReplicas`) are healthy, the volume will be detached. - By the Longhorn UI 1. Access the Longhorn UI and navigate to the `Volume` page. 2. Select the volume that needs offline replica rebuilding. 3. Click on the `Operation` dropdown and click on `Offline Replica Rebuilding`. 4. A UI lightbox will pop out and users choose the option `enabled`. Three options for the `Offline Replica Rebuilding` (`Volume.Spec.OfflineRebuilding`): - `ignored`: The default value. When the `Volume.Spec.OfflineRebuilding` is `ignored`, the offline rebuilding will follow the value of the global setting `offline-replica-rebuilding`. - `enabled`: The offline rebuild process will always start when the volume is detached and degraded. - `disabled`: The offline rebuild process will never start. - By `kubectl` command: 1. Use the command `kubectl -n longhorn-system edit volume [volume-name]` 2. Update the field `Volume.spec.offlineRebuilding` to `enabled`. ```yaml apiVersion: longhorn.io/v1beta2 kind: Volume metadata: ... name: [volume-name] namespace: longhorn-system ... spec: ... numberOfReplicas: 3 offlineRebuilding: enabled ... ``` #### Enable Global Volumes Offline Rebuilding When users want to enable the offline rebuilding of detached volumes: - By the Longhorn UI 1. Access the Longhorn UI and navigate to the `Setting` > `General` page. 2. Check the setting `Offline Replica Rebuilding`. 3. Click the bottom `Save` button. - By `kubectl` command: ```shell kubectl -n longhorn patch setting offline-replica-rebuilding --type=merge -p '{"value": "true"}' ``` When there are detached volumes degraded, the offline rebuilding process of these volumes will start. #### The CSI Attach Request During Volume Offline Rebuilding When the offline rebuilding process is in progress: 1. Users try to start the workload with the Longhorn volume that is in offline rebuilding process. 2. When The CSI attaching volume request is received and the CSI VA ticket is created, the volume rebuild VA ticket will be preempted by the CSI VA ticket. 3. The offline rebuilding process will be canceled because the volume rebuild VA ticket is preempted. 4. After offline rebuilding process is canceled, the volume will be attached to the requested node for the CSI VA ticket. 5. The workload can start to use the volume after the volume attachment. #### Cluster Goes Down And Comes Back Up During Volume Offline Rebuilding Users should be aware that the rebuilding process will restart if it does not finish before the cluster goes down. #### A Worker Node Down When Volume Is Detached - A worker node that contains the volume replica is down. - When offline rebuilding is enabled (either by the global setting or `Volume.Spec.OfflineRebuilding`), detached volumes will start the offline rebuilding process if the number of healthy replicas does not equal `Volume.Spec.NumberOfReplicas`, and will continue until the numbers match. #### A Worker Node Down During Volume Offline Rebuilding - The offline rebuilding will continue until the number of healthy replicas equals to `Volume.Spec.NumberOfReplicas`. - The detached volumes will start the offline rebuilding process if the number of healthy replicas does not equal to `Volume.Spec.NumberOfReplicas`. #### Volume Becomes Faulted During Offline Rebuilding - The rebuilding process will be canceled. - Users need to resolve the issues causing the volume to be faulted before initiating a salvage. ## Design ### Implementation Overview The volume rebuild VA ticket example: ```yaml apiVersion: longhorn.io/v1beta2 kind: VolumeAttachment ... spec: attachmentTickets: volume-rebuilding-controller-[volume-name]: generation: 0 id: volume-rebuilding-controller-[volume-name] nodeID: workerNode01 parameters: disableFrontend: "any" type: volume-rebuilding-controller volume: [volume-name] status: ... ``` - The `Volume.Spec.OfflineRebuilding` field will `NOT` be changed when the global setting `offline-replica-rebuilding` is modified. #### Global Setting `offline-replica-rebuilding` - The setting `offline-replica-rebuilding` is `true`: 1. The volume rebuild controller will create the VA ticket for attaching the volume that the `Spec.OfflineRebuilding` field is `enabled` or `ignored` and need to be rebuilt. ![volume offline rebuilding volume rebuild controller flow](./image/volume-offline-rebuild-vrb-controller-flow.png) 2. The rebuilding process will be started automatically by the volume controller. 3. The volume rebuild controller will check the volume and replicas of the volume status when the volume is updated. 4. If the rebuilt replicas are healthy (healthy replica count is equal to `Volume.Spec.NumberOfReplicas`), the volume rebuild controller will remove the volume rebuild VA ticket to detach the volume and the process is done. - The setting `offline-replica-rebuilding` is `false`: 1. The volume rebuild controller will check if the `Volume.Spec.OfflineRebuilding` field is `disabled` or `ignored`. 2. The offline rebuilding in progress will be canceled by the volume rebuild controller with removing the volume rebuild VA ticket. #### Manually Enable Offline Rebuilding - The `Spec.OfflineRebuilding` field of the detached volume is set to `enabled` by the Longhorn UI, API or `kubectl` command: ```yaml apiVersion: longhorn.io/v1beta2 kind: Volume ... spec: ... numberOfReplicas: 3 offlineRebuilding: enabled ... ``` 1. The volume rebuild controller will create the VA ticket for attaching the volume if rebuilding is necessary. 2. The rebuilding process will be started automatically by the volume controller. 3. The volume rebuild controller will check the volume and replicas of the volume status when the volume is updated. 4. If the rebuilt replicas are healthy (healthy replica count is equal to `Volume.Spec.NumberOfReplicas`), the volume rebuild controller will remove the volume rebuild VA ticket to detach the volume and the process is done. - The `Spec.OfflineRebuilding` field of the detached volume is set to `disabled` by the Longhorn UI, API or `kubectl` command: ```yaml apiVersion: longhorn.io/v1beta2 kind: Volume ... spec: ... numberOfReplicas: 3 offlineRebuilding: disabled ... ``` 1. The rebuilding process will be canceled if necessary. 2. The volume rebuild controller will remove the VA ticket if necessary. #### Attach Request With High Priority VA Ticket During Offline Rebuilding For instance, CSI VA ticket priority 900 is higher than volume rebuild controller VA ticket priority 800: 1. When The CSI attaching volume request is received and the CSI VA ticket is created, the volume rebuild VA ticket will be preempted by the CSI VA ticket by the volume attachment controller. 2. The volume rebuild controller will remove the volume rebuild VA ticket if there is a CSI VA ticket. 3. The volume rebuild controller will record the cancellation of the volume offline rebuilding. 4. After the volume rebuild VA ticket is removed, then the volume will be attached for the CSI VA ticket. 5. Make an event or logs in the controller to notice users the rebuilding is canceled. #### Handle Cluster Goes Down And Comes Back Up During Offline Rebuilding - The volume rebuild controller will examine all volumes: - If `Volume.Spec.OfflineRebuilding` is `disabled`, remove the volume rebuild VA ticket for the volume. - If `Volume.Spec.OfflineRebuilding` is `enabled` or if the global setting `offline-replica-rebuilding` is `true` and the `Volume.Spec.OfflineRebuilding` field is `ignored`, the volume needs to be rebuilt and there is no CSI VA ticket: - Add the volume rebuild VA ticket of the volume if it does not exist. - Check the volume rebuilding status if the volume rebuild VA ticket exists. #### Volume Faulted During Offline Rebuilding 1. The volume rebuild controller will remove the volume rebuild VA ticket for stopping the rebuilding process. 2. The volume rebuild controller will record the cancellation of the volume offline rebuilding. 3. Make an event or logs in the controller to notice users the rebuilding is canceled. ### Test plan - The replica count is less than the number of replicas in the `Volume.Spec`: - Set the field `Volume.Spec.OfflineRebuilding` to `enabled` 1. Create a volume with 3 replicas in a 3 worker nodes cluster and write some data to the volume. 2. Detach the volume. 3. Delete a replica of the volume. 4. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 5. Wait for the volume detached. 6. Check if healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. 7. Check if the `Volume.Spec.OfflineRebuilding` is still `enabled`. 8. Delete a replica of the volume. 9. The offline rebuilding will start again. 10. Wait for the volume detached. 11. Check if healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. 12. Check if the `Volume.Spec.OfflineRebuilding` is still `enabled`. - Set the global setting `offline-replica-rebuilding` to `true` 1. Create a volume with 3 replicas in a 3 worker nodes cluster and write some data to the volume. 2. Detach the volume. 3. Delete a replica of the volume. 4. Set the global setting `offline-replica-rebuilding` to `true` to enable the offline rebuilding. 5. Wait for the volume detached. 6. Check if healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. 7. Check if the `Volume.Spec.OfflineRebuilding` is still `ignored`. 8. Delete a replica of the volume. 9. The offline rebuilding will start again. 10. Wait for the volume detached. 11. Check if healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. 12. Check if the `Volume.Spec.OfflineRebuilding` is still `ignored`. 13. Set the global setting `offline-replica-rebuilding` to `false`. 14. Check if the `Volume.Spec.OfflineRebuilding` is `ignored` 15. Set the `Volume.Spec.OfflineRebuilding` to `disable`. 16. Delete a replica of the volume. 17. Set the global setting `offline-replica-rebuilding` to `true`. 18. The offline rebuilding will not start. 19. Check if the `Volume.Spec.OfflineRebuilding` is still `disable`. 20. Set the global setting `offline-replica-rebuilding` to `false`. 21. Check if the `Volume.Spec.OfflineRebuilding` is still `disable`. - Enable the global setting `offline-replica-rebuilding` when many volumes are degraded 1. Create 10 volumes with 3 replicas in a 3 worker nodes cluster and write some data to all volumes. 2. Detach all volumes. 3. Delete a replica of all volumes. 4. Set the global setting `offline-replica-rebuilding` to `true` to enable the offline rebuilding. 5. Wait for that volumes are attached. 6. Patch the global setting `concurrent-replica-rebuild-per-node-limit` to 3 7. Check if only 3 rebuilding processes start. 8. Wait for all volumes are detached. 9. Check if healthy replicas count of all volumes equals to `Volume.Spec.NumberOfReplicas`. - The CSI attaching request should preempt the offline rebuilding process: 1. Create a workload with Longhorn volume having 3 replicas in a 3 worker nodes cluster 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Delete a replica of the volume. 5. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 6. When the rebuilding is in progress, scale up the workload. 7. Check if the offline rebuilding process is canceled. 8. Check if the workload is working well. - The one worker node shutdown/reboot during offline rebuilding in the cluster with 3 worker nodes: 1. Create a workload with Longhorn volume with 3 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Delete a replica of the volume. 5. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 6. Shutdown/reboot a worker node. 7. Check if the volume offline rebuilding is in progress. 8. The volume is detached after healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. - The one worker node shutdown in the cluster with 4 worker nodes: 1. Create a workload with Longhorn volume with 3 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 5. Shutdown a worker node that contain a volume replica. 6. Check if the volume offline rebuilding is in progress. 7. The volume is detached after healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. - The cluster with 3 worker nodes goes down and comes back up during offline rebuilding: 1. Create a workload with Longhorn volume with 3 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Delete a replica of the volume. 5. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 6. Shutdown the cluster. 7. Bring the cluster up. 8. Check if the volume offline rebuilding is restarted. 9. Check if the volume offline rebuilding is done 10. The volume is detached. - The source replica is deleted during offline rebuilding: 1. Create a volume with 3 replicas in a 3 worker nodes cluster and write some data to the volume. 2. Detach the volume. 3. Delete a replica of the volume. 4. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 5. Check if the rebuilding process is start and the new replica is replenished. 6. Delete or crash the source replica process. 7. Check if the volume rebuilding is still working. 8. Check if the volume offline rebuilding is done 9. The volume is detached. ### Upgrade strategy - The `Volume.Spec.OfflineRebuilding` field of all volumes will be updated to `ignored`. ================================================ FILE: enhancements/20250508-v2-volume-cloning.md ================================================ # V2 Volume Cloning ## Summary Support volume cloning feature for v2 data engine volumes ### Related Issues https://github.com/longhorn/longhorn/issues/7794 ## Motivation ### Goals This ticket has 2 goals: 1. Regular cloning in which the data is decoupled from the source volume. 1. Fast cloning in which data is depended on the source volume. There should be zero data copy in this case and clone should be instant. Optionally, this feature can also serve as the foundation for v2 volume exporting/downloading ## Proposal ### User Stories #### Story 1 - Regular v2 cloning User are using Longhorn v2 data engine volumes. They want to create a new independent volume from a snapshot of the source volume. Before this feature, the user can only do it with v1 data engine volume. Main use case for this one is general CSI cloning and Harvester VM image feature. #### Story 2 - Fast v2 cloning User are using Longhorn with some external backup solution such as Kasten or Velero. These solution want to take a snapshot of a volume. Then quickly clone that snapshot into a new PVC. Then read data of the new snapshot and upload it to backup store such as S3 object store. The fast cloning feature will help improve efficiency for this use case. ### User Experience In Detail ### API changes #### Longhorn API Add a new field into `volume.Spec` to indicate whether user wants a fast cloning or regular cloning for a new volume provisioned from a snapshot of other volume. For example: ```go type VolumeSpec struct { // +optional DataSource VolumeDataSource `json:"dataSource"` // +kubebuilder:validation:Enum=fast;full;decoupled // +optional CloneMode bool `json:"cloneMode"` } ``` With: 1. `CloneMode: fast`: Longhorn should perform fast clone 1. `CloneMode: full`: Longhorn should perform regular clone 1. `CloneMode: decoupled`: If the user initially clone the volume by fast clone, the volume are dependent on the source volume's snapshot. There are a few restriction for a volume in this mode (see below section for more detail). The user can change the mode to `decoupled` which will make the volume become independent of the source snapshot. #### SPDK API Implement a new SPDK feature/APIs: 1. `bdev_lvol_start_deep_copy` will transfer the clusters which are either owned by the target snapshot or its parent. Therefore, the new cloning volume are kept thin provisioned. 1. `bdev_lvol_check_deep_copy` will check the status of the deep copy operation ## Design ### Regular v2 cloning Requirements and assumptions: 1. The data is decoupled from the source volume 1. The newly target volume can have multiple replicas 1. The replicas of the target volume can live in different nodes/disks than the replicas of the source volume 1. Should consume minimal space / network bandwidth / CPU consumption during and after the cloning Main use case for this one is general CSI cloning and Harvester VM image feature: * User want to create a new PVC from a CSI snapshot * Harvester want to create a new root disk from a golden image. This will integrate well with the new Harvester's general image management feature. A.K.A This can serve as an alternative to the Longhorn v2 backing image feature Flow: 1. User create a new volume with: ```yaml volumeSpec: dataSource: "snap://source-volume/source-snapshot-1" cloneMode: "full" ``` 1. Longhorn manager create a new volume with 1 replica and schedule the replica to a suitable node 1. Longhorn manager send the clone GRPC to the replica server 1. Replica server then call SPDK API `bdev_lvol_start_deep_copy` (see [definition of this API above](#spdk-api)) to copy the data from the source snapshot `source-snapshot-1` into the new snapshot of the replica 1. Replica server periodically check the status of the transfer using `bdev_lvol_check_deep_copy` (see [definition of this API above](#spdk-api)) SPDK API 1. Once the data was transfer replica server reloads 1. Volume controller then rebuild more replica for the volume to meet the HA requirement ### Fast v2 cloning Requirements and assumptions: 1. There should be zero data copy in this case and clone should be instant 1. The data is depended on the source volume 1. The newly target volume can only have 1 replica. And the replica of the target volume must be on same disk as one of the replica of the source volume 1. Replica cannot move to new node and cannot be rebuilt 1. All restriction can be removed if the user decouple the volume (see [Decoupled a fast v2 cloned volume](#decoupled-a-fast-v2-cloned-volume) ) Main use case for this is external backup operation like Kasten or Velero which only need to read the data once and delete the cloned volume Flow: 1. User create a new volume with: ```yaml volumeSpec: dataSource: "snap://source-volume/source-snapshot-1" cloneMode: "fast" ``` 1. Longhorn manager create a new volume with 1 replica and schedule the replica to the node which has a healthy replica of the source volume 1. Longhorn manager send the clone GRPC to the replica server 1. Replica server then call SPDK API `bdev_lvol_snapshot` to create new snapshot `source-snapshot-1-clone-entry-point` from source snapshot `source-snapshot-1` 1. Replica server then call SPDK API `bdev_lvol_clone` to create new lvol as a clone of the snapshot `source-snapshot-1-clone-entry-point` 1. Replica server reloads 1. Clone finished 1. The source volume should ignore the all snapshot with the name `*-entry-point` and its children when building snapshot tree or rebuilding See more details about the flow at [here](./assets/v2-volume-cloning/longhorn-v2-fast-cloning.pdf) ### Decoupled a fast v2 cloned volume Requirements and assumptions: 1. A volume is create in fast clone mode 1. It has [restrictions](#fast-v2-cloning) above 1. If wanted, user can remove the restriction by performing volume decouple Flow: Continue from the flow in the section [Fast v2 cloning](#fast-v2-cloning) above 1. User update the volume with: ```yaml volumeSpec: dataSource: "snap://source-volume/source-snapshot-1" cloneMode: "decoupled" ``` 1. Longhorn manager send the gRPC to the replica server 1. Replica server find the root snapshot which is the child of the snapshot `source-snapshot-1-clone-entry-point` 1. Replica server then call SPDK API `bdev_lvol_decouple_parent` repeatedly to decouple it from all of the parents 1. If the snapshot `source-snapshot-1-clone-entry-point` as no more child, it is removed 1. The decoupling finished 1. All restrictions are removed ### Test plan TODO ### Upgrade strategy No upgrade strategy is need ## Note [optional] Additional notes. ================================================ FILE: enhancements/20250701-configurable-backup-block-size.md ================================================ # Configurable Backup Block Size ## Summary This enhancement adds support for user configured (storage class, volumes) backup block size. ### Related Issues - https://github.com/longhorn/longhorn/issues/5215 ## Motivation ### Goals User is able to configure the backup block size other than the default size 2MiB. ### Non-goals - There are pre-defined options of the valid backup block size. User cannot set the size other than the defined size. - The backup block size is configurable only when creating new volumes. This is immutable after volume is created. - The backup block size is not configurable for backup restore and disaster recovery (DR) volumes. - To the existing backups, the block size is immutable. - If a backup is created using a non-default block size, this backup cannot be restored by the engine that supports only the default block size. ## Proposal Provides two available backup block size to be set in the volume CR: - 2MiB, the default size, and backward compatible with the older version - 16MiB Benchmark: - 3-node K3s cluster locally hosted by Vagrant-libvirt - 5GB v1 volume with 4GB random content single file, v2 engine is disabled in this cluster - Backup target minio is also locally hosted inside the same cluster. | Block Size | Num. of Blk files | Accumulate size on bkup target | Avg Transmission Time | |:----------:|:-----------------:|:------------------------------:|:---------------------:| | 2MiB | 2063 | 4211852 | 29 sec | | 16MiB | 266 | 4197748 | 19 sec | ### User Stories Users can select an appropriate backup block size when creating a volume for their application, based on both the application requirements and the evaluation results regarding the characteristics of the backup target. The larger block size will improve the compression but further also reduce the number of block files, which will potentially lower the cost for the lookups of backup targets. #### Configure The Default Backup Block Size Users can configure global default backup block size setting `default-backup-block-size`. This configuration applies to newly created volumes. #### Configure The Block Size In Storage Class Users can specify the desired backup block size as a parameter in the storage class. This parameter is used when creating volumes. If no backup block size is specified, the system uses the globally configured default value. #### Configure The Block Size In Volume CR A field is available to specify the backup block size when creating a volume CR. This field becomes immutable once the CR is created. If no backup block size is specified, the system uses the globally configured default value. #### Configure The Block Size In Volume Creation UI Users specify the backup block size during the volume creation. It provides the available options to the user, and sets to 2MiB by default. The backup block size is listed in the volume details. While creating the volume form a backup, it also provides the available backup block size options to the user. This option affects the new backup creation for this restored volume. While creating the disaster recovery volume, the backup block size is fixed to the existing backups. #### View The Block Size In Backup List UI Users can check the size from backup list UI, so that they can confirm the backup block size. ## Design ### Setting A new setting `default-backup-block-size` is introduced as the default backup block size for volume creation. The setting is an integer to specify the block size in MB, , one of `2` (2 MiB) or `16` (16 MiB). ### CSI A new parameter `backupBlockSize` is introduced to the storage class. ```yaml kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: pvc-larger-backup-block-size provisioner: driver.longhorn.io parameters: ... backupBlockSize: 16Mi ``` - A string of quantity. - Longhorn accepts only the value of `2Mi` and `16Mi` ### Longhorn Manager - The new string field `BackupBlockSize` is configured in the volume CR and accepts a size integer in bytes, one of `2097152` (2 MiB) or `16777216` (16 MiB). ```go type VolumeSpec struct { ... // BackupBlockSize indicate the block size to create backups. The block size is immutable. // +kubebuilder:validation:Type=string // +kubebuilder:validation:Enum="2097152";"16777216" // +optional BackupBlockSize int64 `json:"backupBlockSize,string"` ... } ``` - If the `BackupBlockSize` is set to `0`, it means to set to the global default block size. - While creating a new backup CR, the new field `BackupBlockSize` is copied from the volume CR. ```go type BackupSpec struct { ... // The backup block size. 0 means the legacy default size 2MiB, and -1 indicate the block size is invalid. // +kubebuilder:validation:Type=string // +kubebuilder:validation:Enum="-1";"2097152";"16777216" // +optional BackupBlockSize int64 `json:"backupBlockSize,string"` ... } ``` - To a backup is fetch from a remote backup store: - If the block size information is unset, it will fall back to legacy 2 MiB size - If the backup is somehow broken and the block size is invalid, the new backup CR will be still created to represent the existence, but a special block size `-1` is used to mark this backup is unusable. - Backup controller creates monitor for the backup CR, and passes the block size to the sync agent by set a new parameter `backup-block-size` in the volume descriptor. #### Webhooks - The volume mutator assigns the global default value specified in setting `default-backup-block-size` when the specified value is `0`. - The volume validator ensures the validity of the new string field `backupBlockSize`, and prohibits modifications. - The block size must be `2097152` (2 MiB) or `16777216` (16 MiB). - The size of the volume must be an integer multiple of the backup block size. - When updating the volume size for expansion, the new size must be an integer multiple of the backup block size. - The backup validator also prohibits modifications to the field `backupBlockSize`, except the original block size is not set. #### Upgrade To the existing volume and backup CRs, the `backupBlockSize` will be set to `2097152` (2 MiB). ### Longhorn Engine / Replica While a sync agent is creating a new backup: - For request initiation, both to the `backup create` CLI subcommand and the gRPC interface, accept the new parameter to specify the backup block size. If the parameter `backup-block-size` is not set or invalid, fallback the block size to the default size `2,097,152` (2 MiB). - The engine then calculates the blocks from the sections using the given backup block size. - While uploading the backup to the target, the engine records the backup block size in number of bytes inside the metadata file (`${volume_name}/backups/backup_*.cfg`) ```json { "Name": "backup-1e0f6f32f3a24aac", "VolumeName": "vol", "SnapshotName": "1853016f-de97-4bc3-b506-22f4a509820b", ... "BlockSize": 2097152, ... } ``` While a replica is restored from a backup: - During the initialization, load the block size from the backup metadata file, and validate the size. The volume size must be an integer multiple of the block size. - To incremental restoring, accept the block size of the new backup that is identical with the last one. - During the restoring, load the blocks in the configured block size. ### Longhorn Manager RESTful API - Add a new field, `BackupBlockSize`, to the volume and backup spec. - While Longhorn manager handles volume creation request, it set the backup block size to the new volume CR. - Don't provide any method to edit the backup block size on existing volume. ### Longhorn UI - Provide option to select the backup block size for volume creation, includes: - Creating a new volume. - Restoring from a backup. - In the backup list page, add a new column for backup block size. ### Test plan #### Longhorn Upgrade While upgrading the Longhorn from v1.9 to v1.10, all existing volume and backup CRs should be updated to use 2 MiB backup block size. #### Global Default Backup Block Size - When a volume is created without specifying the backup block size, the backup block size should be set to the value configured in `default-backup-block-size`. - The change of `default-backup-block-size` does not affect to the existing volumes. #### Create A Volume Using Large Backup Block Size Via Longhorn API The volume's backup block size can be specified while creating it via Longhorn volume creation API. #### Create A Volume Using Large Backup Block Size Via PVC The volume's backup block size can be specified in the storage class, and the volumes in this class should respect the backup block size. #### Volume Backup Creation While creating a backup from a volume, the backup's block size should be identical with the one specified in the volume. #### Remote Backup Fetching Generate a backup on the remote backup store, and specify the block size in the backup's metadata. - The backup CR of the fetched backup should be the one specified in the remote backup metadata. - If the block size is not set in the remote backup metadata, the backup CR's block size should be 2 MiB. - If the block size is invalid in the remote backup metadata, the backup CR's block size should be `-1`. #### Prevent Updating The Block Size - Given a volume, the mutation of the back block size is rejected. - Given a backup, the mutation of the back block size is rejected. ## References - Some reference design by other backup systems: https://github.com/longhorn/longhorn/issues/5215#issuecomment-3051943659 ================================================ FILE: enhancements/20250721-v2-engine-interrupt-mode.md ================================================ # V2 Engine - Interrupt Mode ## Summary This proposal introduces **interrupt mode** support for the Longhorn V2 engine to reduce CPU usage, especially in cluster with idle or low I/O workloads. The current V2 engine relies on SPDK's polling mode, which continuously consumes CPU resources (~100%) even when no I/O is occurring. In the initial phase, interrupt mode will be implemented for the SPDK NVMe/TCP transport using a **hybrid approach**: - The NVMe-oF **target** will use `epoll` to wait for socket readiness events. - The NVMe-oF **initiator** will continue to poll periodically to flush I/O completions. In future iteration, we may explore `epoll` with `eventfd` to detect queue push events and forward I/O commands directly to the socket file descriptor, enabling a more fully even-driven design. ### Related Issues - https://github.com/longhorn/longhorn/issues/9834 - ~https://github.com/longhorn/longhorn/issues/9419~ ## Motivation Offer users a setting to enable interrupt mode in the Longhorn V2 engine, allowing instance managers to operate with lower CPU consumption. ### Problem The SPDK's polling mode in V2 engine causes near-constant CPU utilization even when volumes are idle, leading to inefficient resource usage. ### Goals - Provide users the ability to **opt-in** to interrupt mode to reduce CPU usage. - Offer configuration for the initiator’s polling interval, balancing CPU consumption with I/O latency (phase 2). ### Non-goals [optional] - Interrupt mode support for other transport protocols (e.g., RDMA) is out of scope. - Switch from `spdk_tgt` to `nvmf_tgt`, see GitHub issue [comment](https://github.com/longhorn/longhorn/issues/9419#issuecomment-3101365564). ## Proposal ### Design Overview The initial interrupt mode implementation adopts a **hybrid approach**: - The SPDK NVMe-oF **target** waits on socket readiness using `epoll`. - The SPDK NVMe-oF **initiator** periodically call `bdev_nvme_poll` at an interval to process pending I/O completions. This hybrid approach provides immediate CPU savings while maintaining responsiveness. **Future Investigate** We plan to explore `eventfd` with `epoll` to detect queue push events more precisely and forward I/O command directly to the socket file descriptor. This approach could enable a fully event-driven architecture, potentially reducing CPU overhead further. ### Configurable Settings - `data-engine-interrupt-mode-enabled`: Enables interrupt mode for V2 engine volumes (phase 1). - `data-engine-nvme-ioq-poll-period-us`: Polling interval (microseconds) for initiator I/O queue polling (phase 2). ### User Stories #### Story 1: Reduce CPU Usage **Before:** V2 instance manager uses ~100% CPU even when idle. **After:** With interrupt mode, CPU usage drops dramatically under low or no I/O. #### Story 2: Configurable Performance **Before:** Users can't adjust I/O command polling frequency. **After:** Users can tune `data-engine-nvme-ioq-poll-period-us` setting to balance between CPU consumption and I/O responsiveness. ### User Experience In Detail User enable interrupt mode via global settings: ```yaml defaultSetting: data-engine-interrupt-mode-enabled: true data-engine-nvme-ioq-poll-period-us: 100 # in microseconds (1ms) ``` When enabled: - The NVMe-oF **target** waits on `epoll` for I/O readiness. - The NVMe-oF **initiator** polls periodically to flush I/O completions. - CPU usage decreases, especially in idle or light workload scenarios. ### API changes None. ## Design ### Key Changes 1. **Enable interrupt mode** for the SPDK NVMe-oF TCP target. 1. **Replace busy-wait loops** with pollers to complete TCP connections asynchronously and enable the target to respond to epoll events. 1. **Update the SPDK NVMe bdev module** to: - Periodically poll I/O queues to flush pending initiator commands. - Periodically poll the admin queue to issue keep-alive commands. 1. **Introduce global settings** for the feature support. ### Phased Rollout #### Phase 1: Basic Interrupt Mode - Set a fixed I/O queue polling interval of `100ms` (`nvme_ioq_poll_period_us = 100`), based on [benchmark results](#benchmark-results). - Utilize `g_opts.nvme_adminq_poll_period_us`, set in `longhorn-spdk-engine`, to ensure timely keep-alive command completion. - Introduce `data-engine-interrupt-mode-enabled` global setting to toggle interrupt mode. - Benchmark I/O performance in polling vs. interrupt mode scenarios. ##### Dependencies - Requires `SPDK v25.x` for foundational change of bdev NVMe interrupt-mode support. - Rebase Longhorn’s SPDK fork on `SPDK v25.x`. ##### SPDK 1. **Enable interrupt mode for NVMe/TCP** in the SPDK bdev NVMe module. 1. **Enable periodic polling** of `bdev_nvme_poll` to flush `sock->queued_reqs` into the TCP stack. 1. **Replace busy-wait loop** with asynchronous poller in the TCP connection handling path. This will be removed once the I/O queue state changed from connecting state. 1. Remove `event_iscsi` from `SPDK_LIB_LIST` to prevent unnecessary iSCSI subsystem initialization in `spdk_tgt`. ##### New Global Setting `data-engine-interrupt-mode-enabled`: - **Type:** Boolean - **Default:** `false` (polling mode is the default) ##### Custom Resource Definition (CRD) Add `InterruptModeEnabled` field to `InstanceManager.status.dataEngineStatus.v2` to indicate whether the V2 data engine is running in **interrupt mode** (`true`) or **polling mode** (`false`). This field is managed and updated by the Longhorn Manager. #### Phase 2: Configurable Polling Interval Support dynamic polling interval configuration in the SPDK engine. ##### New Global Setting `data-engine-nvme-ioq-poll-period-us`: - **Type:** Integer - **Range:** >= 1 - **Default:** `100` (microseconds) ##### SPDK Engine Expose `nvme_ioq_poll_period_us` as an externally configurable parameter. ### Test Plan 1. **Performance Testing:** - Compare performance in polling vs interrupt mode. - Evaluate performance for various polling intervals. 1. **Functional Testing:** - Verify volume creation ensure I/O operations functions correctly with interrupt mode enabled. - Compare CPU usage between polling mode and interrupt mode. 1. **Regression Testing:** - Ensure all V2 volume features behave correctly in both modes. ### Benchmark Results **Environment** - Cloud: AWS - OS: SLES 15 SP7 (AMI: ami-05cf3966cddeb5037) - Kubernetes: k3s v1.32.0+k3s1 - Nodes: t2.2xlarge - Disks: 3× 200GiB EBS gp2 **Interrupt Mode (nvme_ioq_poll_period_us = 1000)** ``` TEST_FILE: /volume/test TEST_OUTPUT_PREFIX: ./test_device TEST_SIZE: 10G MODE: full Benchmarking random read iops Benchmarking random write iops Benchmarking sequential read bandwidth Benchmarking sequential write bandwidth Benchmarking random read latency Benchmarking random write latency ========================= FIO Benchmark Summary For: ./test_device CPU Idleness Profiling: disabled Size: 10G Mode: full ========================= IOPS (Read/Write) Random: 9,077 / 3,000 Bandwidth in KiB/sec (Read/Write) Sequential: 248,535 / 61,601 Latency in ns (Read/Write) Random: 2,010,330 / 3,651,041 ``` **Interrupt Mode (nvme_ioq_poll_period_us = 100)** ``` TEST_FILE: /volume/test TEST_OUTPUT_PREFIX: ./test_device TEST_SIZE: 10G MODE: full Benchmarking random read iops Benchmarking random write iops Benchmarking sequential read bandwidth Benchmarking sequential write bandwidth Benchmarking random read latency Benchmarking random write latency ========================= FIO Benchmark Summary For: ./test_device CPU Idleness Profiling: disabled Size: 10G Mode: full ========================= IOPS (Read/Write) Random: 5,308 / 2,999 Bandwidth in KiB/sec (Read/Write) Sequential: 248,579 / 61,583 Latency in ns (Read/Write) Random: 980,071 / 2,476,177 ``` **Polling Mode** ``` TEST_FILE: /volume/test TEST_OUTPUT_PREFIX: ./test_device TEST_SIZE: 10G MODE: full Benchmarking random read iops Benchmarking random write iops Benchmarking sequential read bandwidth Benchmarking sequential write bandwidth Benchmarking random read latency Benchmarking random write latency ========================= FIO Benchmark Summary For: ./test_device CPU Idleness Profiling: disabled Size: 10G Mode: full ========================= IOPS (Read/Write) Random: 6,996 / 3,007 Bandwidth in KiB/sec (Read/Write) Sequential: 370,937 / 59,414 Latency in ns (Read/Write) Random: 1,288,569 / 2,065,952 ``` **V1 Engine** ``` TEST_FILE: /volume/test TEST_OUTPUT_PREFIX: ./test_device TEST_SIZE: 10G MODE: full Benchmarking random read iops Benchmarking random write iops Benchmarking sequential read bandwidth Benchmarking sequential write bandwidth Benchmarking random read latency Benchmarking random write latency ========================= FIO Benchmark Summary For: ./test_device CPU Idleness Profiling: disabled Size: 10G Mode: full ========================= IOPS (Read/Write) Random: 5,900 / 1,973 Bandwidth in KiB/sec (Read/Write) Sequential: 161,421 / 60,279 Latency in ns (Read/Write) Random: 1,821,355 / 2,468,884 ``` ##### 🔍 Performance Comparison | Mode | Rand Read IOPS | Rand Write IOPS | Seq Read BW (KiB/s) | Seq Write BW (KiB/s) | Rand Read Latency (ns) | Rand Write Latency (ns) | |--------------------------------|----------------|------------------|----------------------|-----------------------|--------------------------|---------------------------| | **V2 (Interrupt, 1000ms)** | **9,077** | 3,000 | 248,535 | **61,601** | 2,010,330 | 3,651,041 | | **V2 (Interrupt, 100ms)** | 5,308 | 2,999 | 248,579 | 61,583 | **980,071** | 2,476,177 | | **V2 (Polling)** | 6,996 | **3,007** | **370,937** | 59,414 | 1,288,569 | **2,065,952** | | **V1 Engine** | 5,900 | 1,973 | 161,421 | 60,279 | 1,821,355 | 2,468,884 | --- Setting `nvme_ioq_poll_period_us` to 100ms (0.1s) seems to offer the best balance between performance and CPU efficiency. It achieves the **lowest read latency** while maintaining reasonable IOPS, making it suitable for general-purpose workloads. By contrast, setting `nvme_ioq_poll_period_us` to 1000ms (1s) increases the wait time before polling for I/O completions. This can allow more I/Os to accumulate and be processed in batches, potentially increasing IOPs. However, it comes at the cost of **higher latency**, making it less suitable for latency-sensitive workloads. ### Upgrade strategy No migration required. Interrupt mode is opt-in and disabled by default, preserving current behavior. ## Note [optional] - https://github.com/longhorn/longhorn/issues/9834#issuecomment-2914384232 ================================================ FILE: enhancements/20250721-v2-volume-expansion.md ================================================ # V2 Volume Expansion (Nvmf frontend) ## Summary Support for user to expand v2 volumes with nvmf frontend. ### Related Issues - https://github.com/longhorn/longhorn/issues/8022 ## Motivation - Promote Longhorn Engine v2 from experimental to production-ready by implementing missing features, improving stability, and achieving functional parity with Engine v1. ### Goals - Allow users to perform online expansion of Engine v2 volumes with the NVMe-oF frontend. - Ensure the expansion workflow and logic are aligned with Engine v1. ### Non-goals - Support for online expansion of v2 volumes with the `ublk` frontend. ## Proposal ### User Stories - V2 volume expansion is needed for general users ### API changes Introduce new SPDK rpc call (1) EngineExpand ```proto rpc EngineExpand(EngineExpandRequest) returns (google.protobuf.Empty); message EngineExpandRequest{ string name = 1; uint64 size = 2; } ``` (2) ReplicaExpand ```proto rpc ReplicaExpand(ReplicaExpandRequest) returns (google.protobuf.Empty); message ReplicaExpandRequest{ string name = 1; uint64 size = 2; } ``` ## Design ### Implementation Overview - **RAID Deletion Before Expansion** - SPDK RAID1 requires equal-sized base bdevs. - If not detach the controller, SPDK's synchronize RPCs during lvol resize may hang due to active frontend references. - **I/O Suspension During Expansion** - During expansion, the engine is locked and I/O is suspended. - Once the expansion completes, I/O is resumed. - **Partial Replica Expansion Handling** - If only some replicas succeed in expansion, the failed replicas are marked as ERR, and the RAID is recreated with the successfully resized replicas. - The expansion still proceeds to ensure minimal disruption, and Longhorn's replica rebuilding mechanism can later recover redundancy. - **Transparent User Experience via NVMe-oF Grace Period** - NVMe-oF allows a grace period during which the device can temporarily disappear without triggering a disconnect on the host side. （`--ctrl-loss-tmo` controller loss timeout, current 30 sec） - In practice, `bdev_lvol_resize` for thin-provisioned volumes is fast. As a result, from the user's perspective, the device remains connected, and the expansion is transparent — they will not notice that the underlying bdev was recreated and reconnected. - **Frontend Recovery Retry on Failure** - In the case of rebuilding frontend failed, the Longhorn manager will continue to retry the expansion process. During the next reconciliation loop, the engine service will attempt to re-establish the frontend connection again. - **Replica State Awareness** - Expansion is skipped if any replica is `rebuilding` or `expanding`. - **Snapshot** - When a snapshot is taken, it retains the original size, even if the parent lvol is later resized. - As a result, performing a `bdev_lvol_clone` from such a snapshot will create a new logical volume with the size of the snapshot, not the expanded size of the original lvol. - For example, if the current volume size is 1GiB and a snapshot named `snapshot-1` is taken, the snapshot will reflect a size of 1GiB. If the volume is then expanded to 2GiB and another snapshot named `snapshot-2` is taken, this snapshot will reflect the new size of 2GiB. **Engine Expand** ``` Expand() │ ├── Lock engine ├── Log "Expanding engine" ├── getReplicaClients() │ └── defer closeReplicaClients() │ ├── requireExpansion(size, replicaClients) │ ├── Check: isExpanding → error │ ├── Check: IsRestoring → error │ ├── Check: SpecSize > size → error │ ├── Check: SpecSize == size → return false │ ├── RoundUp size to MiB → mismatch → error │ ├── Check ReplicaStatusMap is not empty │ ├── Check: all replicas are RW and same size │ ├── Check: currentReplicaSize >= size → error or skip │ └── Mark e.isExpanding = true │ ├── Check: requireExpansion == false │ └── Log and return nil │ ├── prepareRaidForExpansion(spdkClient) │ ├── BdevRaidGet() │ ├── If RAID exists: │ │ ├── If Frontend == SPDKTCP && Endpoint != "": │ │ │ └── initiator.Suspend() → defer Resume() │ │ ├── Else if Frontend == ublk → return error │ │ ├── disconnectTarget(currentTargetAddress) │ │ └── StopExposeBdev() │ │ └── BdevRaidDelete() │ │ ├── NoSuchDevice → log and continue │ │ └── If !deleted → return error │ └── Return suspendFrontend, bdevUUID │ ├── expandReplicas(replicaClients, spdkClient, size) │ ├── For each replica in goroutine: │ │ ├── ReplicaGet() │ │ ├── If already at size → return │ │ ├── disconnectNVMfBdev() │ │ ├── ReplicaExpand() │ │ └── connectNVMfBdev() │ │ └── On any error → mark in failedReplica │ ├── wg.Wait() │ ├── If all replicas failed → return error │ ├── If some failed → mark as ModeERR │ └── Log partial failure, return nil │ ├── reconnectFrontend(spdkClient, bdevUUID, allocator) │ ├── Build replicaBdevList (only healthy ModeRW replicas) │ ├── If empty → return error │ ├── BdevRaidCreate() │ ├── retry.BdevRaidGet() with backoff │ ├── If Frontend == SPDKTCP: │ │ ├── get pod IP │ │ ├── checkInitiatorAndTargetCreationRequirements() │ │ └── handleNvmeTcpFrontend() │ └── Else if ublk → return unsupported error │ ├── defer: │ ├── If retErr: │ │ ├── Log error │ │ ├── Set lastExpansionError, lastExpansionFailedAt │ │ ├── Set e.State = Error, e.ErrorMsg │ │ └── UpdateLogger(replicaStatusMap) │ └── finishExpansion(expanded, size, err) │ ├── If expanded: │ │ ├── Log success │ │ └── Update SpecSize │ └── Else: │ └── Log failure │ └── return nil or wrapped error ``` **Replica Expand** ``` Expand() │ ├── Lock replica │ ├── fetchClusterSize(spdkClient) │ └── If error → return wrapped error │ ├── RoundUp(size, clusterSize) │ └── If roundedSize ≠ size → return error │ ├── Check SpecSize: │ ├── If SpecSize > size → return error │ ├── If SpecSize == size → log and return │ ├── If IsExposed: │ ├── StopExposeBdev(NQN) │ ├── If error ≠ NoSuchDevice → return error │ ├── Set IsExposed = false │ └── reExposeBdev = true │ ├── BdevLvolResize(alias, size) │ └── If !resized || err → verify actual lvol size │ ├── BdevLvolGetByName(alias) │ │ └── If error → return wrapped error │ ├── If lvol.SpecSize ≠ size: │ │ ├── If err → return wrapped error │ │ └── If !resized → return error │ └── Else → log success (despite earlier error) │ ├── If reExposeBdev: │ ├── Generate random NGUID │ ├── StartExposeBdev(NQN, UUID, NGUID, IP, Port) │ └── If error → return │ └── Set IsExposed = true │ ├── Cleanup: │ ├── Set Head = nil │ └── If last ActiveChain == r.Name → remove from chain │ ├── updateHeadCache(spdkClient) │ └── If error → return │ ├── Log "Expanding replica complete" ├── Set SpecSize = size └── Return nil ``` ### V1 Expansion Result | Scenario | expansionSuccess | outOfSyncErrs | Behavior | |--------------------------------------|------------------|---------------|-------------------------------------------------------------------------| | All replicas succeed | true | nil | Expand frontend | | Partial replicas succeed | true | present | Set failed replicas to `ERR`, continue with frontend expansion | | All fail but rollback succeeds | false | nil | Do not expand frontend, no replicas marked `ERR` | | All fail and some rollback also fails | false | present | Some replicas marked `ERR`, do not expand frontend | ### V2 Expansion Result | Scenario | expansionSuccess | outOfSyncErrs | Behavior | |--------------------------------------|------------------|---------------|-------------------------------------------------------------------------| | All replicas succeed | true | nil | Expand frontend | | Partial replicas succeed | true | present | Set failed replicas to `ERR`, continue with frontend expansion, only contain the expanded replica | | All fail | false | nil | Recover frontend, no replicas marked `ERR` | > **Note:** The engine uses RAID 1, which requires all base lvols to be the same size. > To allow partial replica expansion success, the engine will exclude any failed replicas when recreating the RAID bdev. ### Test plan Prepare - Create a volume via the UI - Use at least 2 replicas - Select V2 Data Engine - Choose Block Device as the frontend (NVMe-oF) - Attach the volume to a node **Live Expansion** 1. Perform I/O on the volume (fio to the dm directly) 2. Trigger live volume expansion during the I/O - Expand the volume to a larger size via the Longhorn UI or API - Ensure expansion happens while I/O is ongoing 3. Verify expected behavior - No I/O interruption or filesystem error occurs during or after expansion - Volume size is updated correctly inside the guest OS - All healthy replicas remain online - Use `go-spdk-helper` in the container to verify the bdev information **Snapshot Behavior** Suppose the original size is 1GiB, and it is expanded to 2GiB: 1. Take a snapshot before the expansion. 2. Perform the expansion. 3. Take another snapshot after the expansion. 4. Create clones from both snapshots. 5. The first clone should be 1GiB, and the second one should be 2GiB. **Rebuilding Replica + Expand** 1. Perform some I/O on the device-mapper (dm) volume. 2. Force a rebuild to occur. 3. After the rebuild completes, perform the expansion. 4. Perform additional I/O on the volume. 5. Take a snapshot of the engine, which triggers snapshots of the underlying bdev lvols. 6. Compute and get the checksums of these snapshots — they should be identical. ### Upgrade strategy No upgrade strategy is needed ================================================ FILE: enhancements/20250721-volume-offline-rebuilding-with-resource-awareness.md ================================================ # Volume Offline Rebuilding With Resource Awareness and Retry Backoff ## Summary After [[FEATURE] V1 and V2 volume offline replica rebuilding](https://github.com/longhorn/longhorn/issues/8443), users can rebuild replicas when degraded volumes are detached. But the original design did not consider the resources of the cluster including schedulable nodes and disks. This enhancement adds support with resource awareness and retry backoff functionalities of offline replica rebuild for Longhorn volumes. ### Related Issues - https://github.com/longhorn/longhorn/issues/8443 - https://github.com/longhorn/longhorn/issues/11270 ## Motivation ### Goals - When resources are not sufficient, the offline rebuilding will not start. - When the degraded volumes are attached by the offline rebuilding, Longhorn will detach the volume with a backoff mechanism if resources are not sufficient. ## Proposal We will add a resource awareness mechanism into the offline rebuilding flow, so it will only mention the modified parts from the [previous proposal](./20250407-volume-offline-rebuilding#proposal). ### User Stories - Users want to automatically rebuild replicas while a degraded volume is detached to ensure maintain data redundancy. - A worker node is up with the offline rebuilding enabled. Longhorn will check if resources are sufficient to trigger the offline rebuilding for degraded volumes. - A worker node is down during the offline rebuilding process. Longhorn will check if resources are sufficient for rebuilding to stop the offline rebuilding. Then users will not have volumes always stuck in attached for unfinishable rebuilding. ### User Experience In Detail | `Volume.Spec.OfflineRebuilding` \ setting `offline-replica-rebuilding` | `true` | `false` | | :---: | :---: | :---: | | `ignored` | ✓ | X | | `enabled` | ✓ | ✓ | | `disabled` | X | X | - ✓: The offline rebuilding is enabled. - X: The offline rebuilding is disabled. #### Enable Global Volumes Or An Individual Volume Offline Rebuilding When offline rebuilding is enabled and volumes are degraded, the degraded volumes will be attached until the volume becomes healthy from the [previous design](./20250407-volume-offline-rebuilding#manually-enable-an-individual-volume-offline-rebuilding). It is not reasonable to keep repeating attaching degraded volumes if the resources are insufficient, for example, only 2 schedulable nodes or disks for 3 replicas. Therefore, before the volume rebuild controller starts to attach the volume, it will check if there are the disk candidates. If there is no disk candidate, the replica rebuilding will not start until the resources are sufficient. #### A Worker Node Down During Volume Offline Rebuilding If a worker node goes down and resources become insufficient (the number of available nodes or disks are less than the number of the replica of volume), users must wait for an interval to confirm that the resources are indeed unavailable for rebuilding the replica. Longhorn will have a replica scheduler to check if there is a failed reusable replica or a disk candidate for rebuilding a replica. The maximum interval will be `replica-replenishment-wait-interval` if the replica scheduler tries to replenish a new replica and fails. If the volume condition `Scheduled` becomes `false`, the volume will be detached automatically. #### A Worker Node Up When Volume Offline Rebuilding Enabled When the offline rebuilding is enabled and a worker node get ready in the cluster, Longhorn will immediately use the replica scheduler to check if the node and its disk are schedulable for degraded volumes. If yes, the degraded volumes will be attached to rebuild the replica. After the replica on this node is rebuilt, the volume will be detached if it is healthy or wait for an interval (the maximum interval will be `replica-replenishment-wait-interval`) to confirm that the resources are insufficient for rebuilding another replica and then the volume will be detached as well. ## Design Add a resource awareness and a backoff retry mechanisms into the offline rebuilding flow, so here will only mention the modified parts from the [previous design](./20250407-volume-offline-rebuilding#design). ### Resource Awareness Before creating the volume rebuild VA ticket to attach the volume, the volume rebuild controller will check if there is a reusable replica or a disk candidate: ```golang import "github.com/longhorn/longhorn-manager/scheduler" ... func NewVolumeRebuildingController(...) (*VolumeRebuildingController, error) { ... vbc := &VolumeRebuildingController{ ... } vbc.scheduler = scheduler.NewReplicaScheduler(ds) ... } ... func (vbc *VolumeRebuildingController) syncLHVolumeAttachmentForOfflineRebuild(...) (...) { ... if !vbc.isVolumeReplicasHealthy(vol.Spec.NumberOfReplicas, replicas) { ... replicaReusableOrSchedulable, err := vbc.isVolumeReplicasReusableOrSchedulable(vol, replicas) ... if !replicaReusableOrSchedulable { // There is no reusable replica or disk candidate to schedule a new replica return va, nil } createOrUpdateAttachmentTicket(va, attachmentID, vol.Status.OwnerID, longhorn.AnyValue, longhorn.AttacherTypeVolumeRebuildingController) } ... } ... func (vbc *VolumeRebuildingController) isVolumeReplicasReusableOrSchedulable(vol *longhorn.Volume, rs map[string]*longhorn.Replica) (bool, error) { reusableFailedReplica, err := vbc.scheduler.CheckAndReuseFailedReplica(rs, vol, "") ... if reusableFailedReplica != nil { // rebuild this reusable replica first return true, nil } ... replicaDiskCandidates, multiError, err := vbc.scheduler.FindDiskCandidates(replica, rs, vol) ... return len(replicaDiskCandidates) > 0, nil } ``` ### Backoff Retry Use the volume condition `Scheduled` to check if a replica can be schedulabld to a node or a disk candidate for rebuilding. When the volume condition `Scheduled` becomes `false`, it means a new replica failed to be scheduled to a disk candidate and resources are insufficient. In general rebuilding process, the failed reusable replica will have a rebuild retry count and a backoff mechanism (initialized from 1 minute with maximum back off interval 3 minutes.) ```golang func (c *VolumeController) replenishReplicas(...) ...{ ... reusableFailedReplica, err := c.scheduler.CheckAndReuseFailedReplica(rs, v, hardNodeAffinity) ... if reusableFailedReplica != nil { if !c.backoff.IsInBackOffSinceUpdate(reusableFailedReplica.Name, time.Now()) { ... if datastore.IsReplicaRebuildingFailed(reusableFailedReplica) { reusableFailedReplica.Spec.RebuildRetryCount++ } c.backoff.Next(reusableFailedReplica.Name, time.Now()) continue } ... } ... ``` If there are no failed reusable replicas or rebuilding failed reused replicas failed, and the `replica-replenishment-wait-interval` is up (depend on the `Volume.Status.LastDegradedAt`), the volume controller will try to create a new replica to rebuild the volume. ```golang func (c *VolumeController) replenishReplicas(...) ...{ ... if checkBackDuration := c.scheduler.RequireNewReplica(rs, v, hardNodeAffinity); checkBackDuration == 0 { newReplica := c.newReplica(v, e, hardNodeAffinity) if hardNodeAffinity == "" { if multiError, err := c.precheckCreateReplica(newReplica, rs, v); err != nil { ... v.Status.Conditions = types.SetCondition(v.Status.Conditions, longhorn.VolumeConditionTypeScheduled, longhorn.ConditionStatusFalse, longhorn.VolumeConditionReasonReplicaSchedulingFailure, aggregatedReplicaScheduledError.Join()) continue } } if err := c.createReplica(newReplica, v, rs, !newVolume); err != nil { return err } } else { // Couldn't create new replica. Add the volume back to the workqueue to check it later c.enqueueVolumeAfter(v, checkBackDuration) } } ... } ``` When there is a volume rebuild VA ticket and the volume is attached, the volume rebuild controller will check if the rebuilding is starting by the volume condition `Scheduled`. Volume will be detached and if the volume condition `Scheduled` is `False`. ```golang ... func (vbc *VolumeRebuildingController) reconcile (...) ... { ... if vbc.isVolumeReplicasRebuilding(vol, engine) { deleteVATicketRequired = types.GetCondition(vol.Status.Conditions, longhorn.VolumeConditionTypeScheduled).Status == longhorn.ConditionStatusFalse return nil } } ... ``` ### Test plan - A worker node goes down after enabling offline rebuilding in the cluster with 3 worker nodes: 1. Create a workload with Longhorn volume with 3 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 5. Shutdown a worker node. 6. Check if the volume offline rebuilding is not triggered. - A worker node goes down during offline rebuilding in the cluster with 3 worker nodes: 1. Create a workload with Longhorn volume with 3 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Delete a replica of the volume. 5. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 6. Wait for volume rebuilding starts. 7. Shutdown a worker node. 8. Check if the volume is still attaching. 9. The volume is detached and still degraded once volume condition `Scheduled` is false. - A new worker node is up and added into the cluster with 3 worker nodes: 1. Create a workload with Longhorn volume with 4 replicas. 2. Write some data to the volume. 3. Scale down the workload to detach the volume. 4. Enable the offline rebuilding by the API `volume.offlineReplicaRebuilding`. 5. Check if the volume is still detached. 6. Bring up a new worker node and check if the volume is attached and rebuilding starts. 7. The volume is detached after healthy replicas count of the volume equals to `Volume.Spec.NumberOfReplicas`. ================================================ FILE: enhancements/20251001-replica-balance-scheduling.md ================================================ # Replica Balance Scheduling ## Summary When scheduling replicas, Longhorn currently selects a disk with the most usable storage. This simple heuristic can lead to unbalanced storage usage across nodes and disks, reducing flexibility for future scheduling and potentially concentrating replicas on fewer nodes. This proposal introduces a `balance-aware` disk selection policy. Instead of always picking the disk with `maximum free space`, the scheduler simulates placing a replica and chooses the placement that results in the most balanced distribution of usable storage: - First across nodes - Then across disks within the selected node This ensures that storage capacity remains evenly utilized, improving fault tolerance and long-term stability. ### Related Issues - https://github.com/longhorn/longhorn/issues/10512 ### Goals - Improve disk selection by introducing a `balance-aware` scheduling algorithm, reducing uneven storage usage across nodes and disks. ### Non-goals - Provide a **perfectly** balanced global scheduling strategy. Replica scheduling in Longhorn involves multiple factors such as: - Tag matching - Replica anti-affinity rules - Node/disk readiness - Reserved/over-provisioned capacity - This proposal only improves the disk selection stage once other filters are applied. The new balance-aware method tries to pick the most balanced disk within the eligible candidates. ## Proposal Design ### Original #### Max Usable Storage: - `Disk_Usable_Storage = (Disk.StorageAvailable - Disk.StorageReserved)` ### Formula #### 1. Balance Factor Formula by Usable Storage: - `Disk_Usable_Storage = (Disk.StorageAvailable - Disk.StorageReserved) - Disk.StorageScheduled` - Lower score = more balanced distribution. - Formula: $$\text{BalanceScore}(X) = \frac{\max(X) - \min(X)}{\text{mean}(X)}$$ Where: - X = set of usable storage values (per node or per disk). #### 2. Balance Factor Formula by Usable Storage Ratio: - `Disk_Usable_Storage = (Disk.StorageAvailable - Disk.StorageReserved) - Disk.StorageScheduled)` - `Disk_Total = Disk.StorageMaximum - Disk.StorageReserved` - Formula: $$\text{usableRatio}(\text{disk}) = \dfrac{\text{usable}(\text{disk})}{\text{totalCapacity}(\text{disk})}$$ $$\text{BalanceScore}(X) = \frac{\max(\text{usableRatio}(X)) - \min(\text{usableRatio}(X))}{\text{mean}(\text{usableRatio}(X))}$$ Where: - X = set of usable storage ratio values (per node or per disk). #### 3. Balance Factor Formula by Hybrid: - Formula: $$ \text{Score} = \alpha \cdot \text{BalanceScore(Absolute)} + (1 - \alpha) \cdot \text{BalanceScore(Ratio)} $$ Where $\alpha \in [0,1]$ controls the trade-off: - $\alpha = 1$: use absolute balance only - $\alpha = 0$: use ratio balance only - $\alpha = 0.5$: balanced mix (equal weight) **Comparison Table** - Disk A: 2 TB total, 200 GiB free → 10% free - Disk B: 200 GiB total, 40 GiB free → 20% free | Formula | Pros | Cons | |-------------------------|----------------------------------------------------------------------|----------------------------------------------------------------------| | **Absolute Usable** | - Ensures large absolute free space remains
- Prevents "out of space" errors | - Ignores disk size differences
- lead to very uneven utilization ratio across disks (Prefer A) | | **Usable Ratio (%)** | - Fairness across heterogeneous disk sizes
- Keeps utilization percentages aligned | - May over-favor large disks
- May reduce absolute headroom on smaller disks (Prefer B) | | **Hybrid (Weighted Mix)** | - Combines absolute and ratio awareness
- Tunable trade-off α | - More complex and computation
- Needs clear default α to avoid confusion | >Note: Currently, the first formula (α = 1) is selected. Each of the above formulas performs differently under various scenarios. In general, the first one should be suitable for most cases. If different requirements arise, we can easily switch to another formula by adjusting α in the future. ### Decision Tree Schema 1. First balance across nodes, 2. Then balance across disks within that node. ``` getDiskWithMostBalanceScore(candidateDisks, replicaSize) ├── compute usable/total storage maps │ ├── nodeUsableStorage / nodeTotalStorage │ └── diskUsableStorage / diskTotalStorage ├── selectBestNode │ ├── simulate placing replica on each node │ ├── computeHybridBalanceScore() │ │ ├── computeBalanceScoreFromUsableStorage() │ │ └── computeBalanceScoreFromUsableStorageRatio() │ └── pick node with lowest imbalance score ├── selectBestDisk │ ├── simulate placing replica on each disk (within best node) │ ├── computeHybridBalanceScore() │ └── pick disk with lowest imbalance score └── return bestDisk ``` ### Example Scenario 1 Using **Balance Factor Formula by Usable Storage** as sample Suppose the cluster has two nodes, and each node contains two candidate disks that are eligible for replica scheduling.Replica size = 100 GiB. | Node | Disk | Usable Storage (GiB) | | ------ | ---- | -------------------- | | Node A | A1 | 900 | | Node A | A2 | 100 | | Node B | B1 | 600 | | Node B | B2 | 700 | **Old Algorithm (max usable space)** - Picks Disk A1 (900 GiB). - Before placement: - Node A total usable = 900 + 100 = 1000 - Node B total usable = 600 + 700 = 1300 - After placement: - Disk A1 usable = 900 - 100 = 800 GiB - Disk A2 usable = 100 - Node A total usable = 800 + 100 = 900 - Node B total usable = 600 + 700 = 1300 So → Node A = 900, Node B = 1300 **New Algorithm (balance first)** ***Step 1, Node Selection*** - Simulate placing on Node A - New totals: - Node A = (900 + 100) − 100 = 900 - Node B = (600 + 700) = 1300 - Node balance score: $$ BalanceScore_{NodeA} = \frac{1300 - 900}{\tfrac{1300 + 900}{2}} = \frac{400}{1100} \approx 0.364 $$ - Simulate placing on Node B - New totals: - Node A = (900 + 100) = 1000 - Node B = (600 + 700) - 1000 = 1200 - Node balance score: $$ BalanceScore_{NodeB} = \frac{1200 - 1000}{\tfrac{1200 + 1000}{2}} = \frac{200}{1100} \approx 0.182 $$ Choose Node B (lower score, more balanced). ***Step 2, Disk Selection*** - Simulate placing on B1 (600 → 500) - New disk usable: B1 = 500, B2 = 700 - Disk balance score: $$ BalanceScore_{NodeA} = \frac{700 - 500}{\tfrac{700 + 500}{2}} = \frac{200}{600} \approx 0.333 $$ - Simulate placing on B2 (700 → 600) - New disk usable: B1 = 600, B2 = 600 - Disk balance score: $$ BalanceScore_{NodeA} = \frac{600 - 600}{\tfrac{600 + 600}{2}} = \frac{0}{600} \approx 0 $$ Choose Disk B2 (perfect balance between B1 and B2). ### Example Scenario 2 | Node | Disk | Usable Storage (GiB) | Total Capacity (GiB)* | Usable % | | ------ | ---- | -------------------- | --------------------- | -------- | | Node A | A1 | 900 | 1000 | 90% | | Node A | A2 | 100 | 200 | 50% | | Node B | B1 | 600 | 800 | 75% | | Node B | B2 | 700 | 1000 | 70% | **Comparison Table** | Placement | Absolute Score | Ratio Score | Hybrid (α=0.5) | | --------- | -------------- | ----------- | -------------- | | B1 | 1.455 | 0.587 | 1.021 | | B2 | 1.455 | 0.582 | 1.019 | For this case, all three formulas, Disk B2 remains the best choice. ### Test plan 1. Prepare 2 Disks with size 40Gi and 25Gi 2. Create Volume 8Gi with 1 Replica 3. Create Volume 8Gi with 1 Replica 4. Create Volume 8Gi with 1 Replica 5. Create Volume 8Gi with 1 Replica 6. Check the replica distribution, comparing to the original one. The result should be more balanced. ### Upgrade strategy No upgrade strategy is needed ================================================ FILE: enhancements/20251017-rwx-volume-endpoint-network.md ================================================ # RWX Volume Endpoint Network ## Summary This proposal introduces the `endpoint-network-for-rwx-volume` setting, allowing users to assign a dedicated Network Attachment Definition (NAD) for RWX (ReadWriteMany) volume endpoints. This improves network isolation for RWX NFS traffic, keeping it separate from the primary Longhorn networks (cluster and storage networks). ### Related Issues - https://github.com/longhorn/longhorn/issues/10269 ## Motivation In current versions (<=v1.10), RWX volume endpoints are limited to using either the cluster network or the storage network. Introducing a dedicated endpoint network enables: - Isolation of RWX NFS server and client traffic from the primary Longhorn networks and other application workloads. - Users have more control over network paths for RWX volumes. ### Goals - Allow RWX volume NFS server endpoints to use a dedicated NAD. - Preserve backward compatibility for existing Longhorn systems that use the storage network for RWX volume. ### Non-goals - **Remote mount support:** Using the host network interface directly is outside Kubernetes network namespace. This means the relevant component pods loses cluster DNS resolution. - **StorageClass/Volume-based networking:** Per-SC/volume network configuration is not supported, as the CSI plugin requires full network access to additional networks, and implementing per-SC/volume networking would be complicated. ## Proposal ### User Stories #### Story 1: Improved Network Control And Isolation For RWX Volumes - **Before:** RWX volume endpoints can only use the cluster or storage network. - **After:** RWX volume endpoints can use a dedicated NAD, isolating traffic from the cluster and storage networks. ### User Experience #### Fresh Install - The user creates a dedicated NAD for RWX volume endpoints. - The user sets the `endpoint-network-for-rwx-volume` global setting to the NAD (`/`), following the same convention as `storage-network`. - When a workload pod is created, its RWX volume is mounted via the NFS endpoint using the NAD-assigned IP on the share-manager pod's `lhnet2` interface. - If the `endpoint-network-for-rwx-volume` global setting is not set, Longhorn will default to using the Kubernetes cluster network, keeping the same behavior when `storage-network-for-rwx-volume-enabled=true`. #### Upgrade - If `storage-network-for-rwx-volume-enabled=true`, the manager migrates the setting to `endpoint-network-for-rwx-volume=storage-network`, preserving existing behavior. - If `storage-network-for-rwx-volume-enabled=false`, the manager sets `endpoint-network-for-rwx-volume` to the default (empty) value, keeping behavior unchanged. ### API changes `None` ## Design ### Implementation Overview #### New Global Setting `endpoint-network-for-rwx-volume`: - **Type:** `String` - **Default:** `""` - **DataEngineSpecific**: `false`, Specifies a dedicated network for mounting RWX (ReadWriteMany) volumes. Leave this field blank to use the default Kubernetes cluster network, which behaves the same as when the deprecated `storage-network-for-rwx-volume-enabled` setting is set to `false`. Changing the value of `endpoint-network-for-rwx-volume` only takes effect when RWX volumes are DETACHED. The network interface/NAD is assigned when the share-manager pod is created, so changing the setting while the volume is attached does not update the existing pod. This behavior is identical to the deprecated `storage-network-for-rwx-volume-enabled` setting. #### Deprecate Old Global Setting - `SettingNameStorageNetworkForRWXVolumeEnabled` will be deprecated. - During upgrade, if the storage network for RWX volumes was previously enabled, the new `endpoint-network-for-rwx-volume` setting will inherit the value of the `storage-network` setting to preserve existing behavior. #### Component CNI - Before this change, CSI plugin and share-manager pods were annotated with `k8s.v1.cni.cncf.io/networks` using the `lhnet1` interface (storage network). - With this change, RWX NFS endpoints use a new interface, `lhnet2`, for network isolation. - Other components unrelated to the NFS endpoint continue to use `lhnet1` for storage network. #### Endpoint Slice The endpoint controller updates the endpoint subset with the share-manager pod's `lhnet2` interface IP from the `k8s.v1.cni.cncf.io/network-status` annotation. ### Test plan 1. **Upgrade from v1.10.x:** Confirm that the `storage-network-for-rwx-volume-enabled` setting is replaced by `endpoint-network-for-rwx-volume`. - If the storage network was previously enabled (`true`), the new `endpoint-network-for-rwx-volume` setting inherits the `storage-network` value. 1. **Feature validation:** Verify that RWX volume mounts function correctly with the endpoint network setting. - Example scenarios: | storage-network | endpoint-network-for-rwx-volume | | --------------- | ------------------------------- | | NAD1 | NAD1 | | NAD1 | NAD2 | | NAD1 | - | ### Upgrade strategy During upgrade, the manager detects the legacy `storage-network-for-rwx-volume-enabled` setting and performs a one-time migration: - If `storage-network-for-rwx-volume-enabled=true`, `endpoint-network-for-rwx-volume` is set to the `storage-network` value. - If the legacy setting is `false` or absent, `endpoint-network-for-rwx-volume` is created with the default (empty) value. ## Note `None` ================================================ FILE: enhancements/20251119-snapshot-heavy-task-concurrency-control.md ================================================ # Snapshot Heavy Task Concurrency Control ## Summary This proposal introduces a new global setting, `snapshot-heavy-task-concurrent-limit`, which allows users to limit the number of concurrently running snapshot heavy tasks—specifically snapshot purge and snapshot clone operations for `v1`. This proposal adds a centralized global concurrency limiter shared across all callers, including controllers, manual API calls, recurring jobs, and CR-based operations. ### Related Issues https://github.com/longhorn/longhorn/issues/11635 ## Motivation Some snapshot operations (such as purge or clone) may trigger internal merge processes that temporarily consume extra disk space. Running many of these tasks concurrently increases the risk of disk exhaustion. A global concurrency limit helps prevent this by controlling how many heavy snapshot tasks can run at the same time. ### Goals Introduce a centralized concurrency controller for snapshot heavy tasks. ### Non-goals - Not intended to limit all snapshot-related actions (e.g., create, list). - Not intended to limit general volume operations such as expansion, rebuild, or backup. ## Proposal Introduce a new component: **SnapshotConcurrentLimiter** This component: - Tracks currently running snapshot heavy tasks (purge / clone). - Enforces the global limit using the new setting. The limiter collects real engine states (purging / cloning) and maintains a synchronized map of active tasks. If the number of active tasks ≥ limit, new tasks are rejected (or delayed by controllers). ## Design ### Implementation Overview New Controller Component: `SnapshotConcurrentLimiter` - Inspect real-time engine purge/clone status - Reject new tasks when exceeding limit - Allow forced bypass if needed #### New Global Setting ```go Name: snapshot-heavy-task-concurrent-limit Type: int Default: 5 Range: >= 0 Category: General Description: Controls how many snapshot purge/clone operations can run concurrently. ``` ### Test plan 1. Set `snapshot-heavy-task-concurrent-limit=1` 2. Set `disable-snapshot-purge=false` 3. Create and Attach a volume 4. IO - sudo ```dd if=/dev/urandom of=/dev/sdb bs=1G count=1 seek=0``` - Take `snapshot 1` - sudo ```dd if=/dev/urandom of=/dev/sdb bs=1G count=20 seek=0``` - Take `snapshot 2` - sudo ```dd if=/dev/urandom of=/dev/sdb bs=1G count=15 seek=10``` - Take `snapshot 3` 5. Remove the snapshot `snapshot 2` -> trigger snapshot purge 6. During the snapshot deletion, try to execute snapshot purge manually ``` curl -X POST \ 'http://localhost:8080/v1/volumes/?action=snapshotPurge' \ -H 'Accept: application/json' ``` it fails with an error: ```cannot start snapshot purge: concurrent snapshot purge limit reached``` 7. Once the snapshot deletion is complete, execute the curl request again. It should succeed. > If you want to increase the purging duration for testing, you can increase the IO workload in `snapshot 2` before triggering the purge operation. ### Upgrade strategy `None` ## Note ### Known Limitation When multiple SnapshotPurge operations are triggered at the same time (such as several CronJobs scheduled on the same interval), any request that exceeds the concurrency limit will be rejected. This can cause certain CronJobs to fail deterministically on every execution. Users are recommended to: - Increase the `snapshot-heavy-task-concurrent-limit` value, or - Schedule snapshot purge CronJobs at different times so they don’t run together. ================================================ FILE: enhancements/YYYYMMDD-template.md ================================================ # Title This is the title of the enhancement. Keep it simple and descriptive. The file name should be lowercased and spaces/punctuation should be replaced with `-`. ## Summary The Summary section is incredibly important for producing high-quality user-focused documentation such as release notes or a development roadmap. A good summary is probably at least a paragraph in length. ### Related Issues The URL For the related enhancement issues in the Longhorn repository. ## Motivation ### Goals List the specific goals of the enhancement. How will we know that this has succeeded? ### Non-goals [optional] What is out of scope for this enhancement? Listing non-goals helps to focus discussion and make progress. ## Proposal This is where we get down to the nitty-gritty of what the proposal actually is. ### User Stories Detail the things that people will be able to do if this enhancement is implemented. A good practice is including a comparison of what the user cannot do before the enhancement is implemented, why the user would want an enhancement, and what the user needs to do after, to make it clear why the enhancement is beneficial to the user. The experience details should be in the `User Experience In Detail` later. #### Story 1 #### Story 2 ### User Experience In Detail Detail what the user needs to do to use this enhancement. Include as much detail as possible so that people can understand the "how" of the system. The goal here is to make this feel real for users without getting bogged down. ### API changes ## Design ### Implementation Overview Overview of how the enhancement will be implemented. ### Test plan Integration test plan. For engine enhancement, also requires engine integration test plan. ### Upgrade strategy Anything that requires if the user wants to upgrade to this enhancement. ## Note [optional] Additional notes. ================================================ FILE: examples/block/block_volume.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-block-vol namespace: default spec: accessModes: - ReadWriteOnce volumeMode: Block storageClassName: longhorn resources: requests: storage: 2Gi --- apiVersion: v1 kind: Pod metadata: name: block-volume-test namespace: default spec: containers: - name: block-volume-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent volumeDevices: - devicePath: /dev/longhorn/testblk name: block-vol ports: - containerPort: 80 volumes: - name: block-vol persistentVolumeClaim: claimName: longhorn-block-vol ================================================ FILE: examples/block/crypto/deployment_with_pvc.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-block-pvc namespace: default spec: accessModes: - ReadWriteOnce volumeMode: Block storageClassName: longhorn-crypto-global resources: requests: storage: 2Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: volume-test labels: app: volume-test namespace: default spec: replicas: 1 selector: matchLabels: app: volume-test strategy: type: Recreate template: metadata: labels: app: volume-test spec: restartPolicy: Always containers: - image: nginx:stable-alpine name: volume-test volumeDevices: - devicePath: /dev/longhorn/testblk name: block-vol volumes: - name: block-vol persistentVolumeClaim: claimName: longhorn-block-pvc ================================================ FILE: examples/block/crypto/secret-crypto-global.yaml ================================================ apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets ================================================ FILE: examples/block/crypto/storageclass-crypto-global.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-global provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" # we currently don't need secrets for volume creation # but it allows for failing the CreateVolume call early # if the required secret has not been setup yet. csi.storage.k8s.io/provisioner-secret-name: "longhorn-crypto" csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-stage-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" # These two are for online expansion of encrypto volumes. # But you need to enable the CSINodeExpandSecret feature gate for # the kube-apiserver and kubelet. For more details, see: # https://kubernetes.io/blog/2022/09/21/kubernetes-1-25-use-secrets-while-expanding-csi-volumes-on-node-alpha/ # csi.storage.k8s.io/node-expand-secret-name: "longhorn-crypto" # csi.storage.k8s.io/node-expand-secret-namespace: "longhorn-system" # we only need crypto keys for node operations, I left these as examples # in case we implement external key vaults in the future # csi.storage.k8s.io/controller-publish-secret-name: "longhorn-crypto" # csi.storage.k8s.io/controller-publish-secret-namespace: "longhorn-system" # csi.storage.k8s.io/controller-expand-secret-name: "longhorn-crypto" # csi.storage.k8s.io/controller-expand-secret-namespace: "longhorn-system" ================================================ FILE: examples/crypto/secret-crypto-customized-rhel-FIPS-enabled.yaml ================================================ --- apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets CRYPTO_KEY_CIPHER: "aes-cbc-essiv:sha256" # this is optional, default value for RHEL CRYPTO_KEY_HASH: "sha256" # this is optional, default value CRYPTO_KEY_SIZE: "256" # this is optional, default value CRYPTO_PBKDF: "pbkdf2" # Only PBKDF2 is supported in FIPS mode, needs to be set on RHEL7 ================================================ FILE: examples/crypto/secret-crypto-customized.yaml ================================================ --- apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets CRYPTO_KEY_CIPHER: "aes-xts-plain64" # this is optional, default value CRYPTO_KEY_HASH: "sha256" # this is optional, default value CRYPTO_KEY_SIZE: "256" # this is optional, default value CRYPTO_PBKDF: "argon2i" # this is optional, default value ================================================ FILE: examples/crypto/secret-crypto-global.yaml ================================================ --- apiVersion: v1 kind: Secret metadata: name: longhorn-crypto namespace: longhorn-system stringData: CRYPTO_KEY_VALUE: "Simple passphrase" CRYPTO_KEY_PROVIDER: "secret" # this is optional we currently only support direct keys via secrets ================================================ FILE: examples/crypto/storageclass-crypto-global.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-global provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" # we currently don't need secrets for volume creation # but it allows for failing the CreateVolume call early # if the required secret has not been setup yet. csi.storage.k8s.io/provisioner-secret-name: "longhorn-crypto" csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-stage-secret-name: "longhorn-crypto" csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" # These two are for online expansion of encrypto volumes. # But you need to enable the CSINodeExpandSecret feature gate for # the kube-apiserver and kubelet. For more details, see: # https://kubernetes.io/blog/2022/09/21/kubernetes-1-25-use-secrets-while-expanding-csi-volumes-on-node-alpha/ # csi.storage.k8s.io/node-expand-secret-name: "longhorn-crypto" # csi.storage.k8s.io/node-expand-secret-namespace: "longhorn-system" # we only need crypto keys for node operations, I left these as examples # in case we implement external key vaults in the future # csi.storage.k8s.io/controller-publish-secret-name: "longhorn-crypto" # csi.storage.k8s.io/controller-publish-secret-namespace: "longhorn-system" # csi.storage.k8s.io/controller-expand-secret-name: "longhorn-crypto" # csi.storage.k8s.io/controller-expand-secret-namespace: "longhorn-system" ================================================ FILE: examples/crypto/storageclass-crypto-per-volume-dedicated-namespace.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-secure-per-volume-ns-longhorn-system provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" # we currently don't need secrets for volume creation # but it allows for failing the CreateVolume call early # if the required secret has not been setup yet. csi.storage.k8s.io/provisioner-secret-name: ${pvc.name} csi.storage.k8s.io/provisioner-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-publish-secret-name: ${pvc.name} csi.storage.k8s.io/node-publish-secret-namespace: "longhorn-system" csi.storage.k8s.io/node-stage-secret-name: ${pvc.name} csi.storage.k8s.io/node-stage-secret-namespace: "longhorn-system" # These two are for online expansion of encrypto volumes. # But you need to enable the CSINodeExpandSecret feature gate for # the kube-apiserver and kubelet. For more details, see: # https://kubernetes.io/blog/2022/09/21/kubernetes-1-25-use-secrets-while-expanding-csi-volumes-on-node-alpha/ # csi.storage.k8s.io/node-expand-secret-name: ${pvc.name} # csi.storage.k8s.io/node-expand-secret-namespace: "longhorn-system" # we only need crypto keys for node operations, I left these as examples # in case we implement external key vaults in the future # csi.storage.k8s.io/controller-publish-secret-name: ${pvc.name} # csi.storage.k8s.io/controller-publish-secret-namespace: "longhorn-system" # csi.storage.k8s.io/controller-expand-secret-name: ${pvc.name} # csi.storage.k8s.io/controller-expand-secret-namespace: "longhorn-system" ================================================ FILE: examples/crypto/storageclass-crypto-per-volume.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-crypto-per-volume provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" encrypted: "true" # we currently don't need secrets for volume creation # but it allows for failing the CreateVolume call early # if the required secret has not been setup yet. csi.storage.k8s.io/provisioner-secret-name: ${pvc.name} csi.storage.k8s.io/provisioner-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-publish-secret-name: ${pvc.name} csi.storage.k8s.io/node-publish-secret-namespace: ${pvc.namespace} csi.storage.k8s.io/node-stage-secret-name: ${pvc.name} csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace} # These two are for online expansion of encrypto volumes. # But you need to enable the CSINodeExpandSecret feature gate for # the kube-apiserver and kubelet. For more details, see: # https://kubernetes.io/blog/2022/09/21/kubernetes-1-25-use-secrets-while-expanding-csi-volumes-on-node-alpha/ # csi.storage.k8s.io/node-expand-secret-name: ${pvc.name} # csi.storage.k8s.io/node-expand-secret-namespace: ${pvc.namespace} # we only need crypto keys for node operations, I left these as examples # in case we implement external key vaults in the future # csi.storage.k8s.io/controller-publish-secret-name: ${pvc.name} # csi.storage.k8s.io/controller-publish-secret-namespace: ${pvc.namespace} # csi.storage.k8s.io/controller-expand-secret-name: ${pvc.name} # csi.storage.k8s.io/controller-expand-secret-namespace: ${pvc.namespace} ================================================ FILE: examples/csi/example_pv.yaml ================================================ apiVersion: v1 kind: PersistentVolume metadata: name: longhorn-vol-pv spec: capacity: storage: 2Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: longhorn csi: driver: driver.longhorn.io fsType: ext4 volumeAttributes: numberOfReplicas: '3' staleReplicaTimeout: '2880' volumeHandle: existing-longhorn-volume --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-vol-pvc namespace: default spec: accessModes: - ReadWriteOnce resources: requests: storage: 2Gi volumeName: longhorn-vol-pv storageClassName: longhorn --- apiVersion: v1 kind: Pod metadata: name: volume-pv-test namespace: default spec: restartPolicy: Always containers: - name: volume-pv-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent livenessProbe: exec: command: - ls - /data/lost+found initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: vol mountPath: /data ports: - containerPort: 80 volumes: - name: vol persistentVolumeClaim: claimName: longhorn-vol-pvc ================================================ FILE: examples/data_migration.yaml ================================================ apiVersion: batch/v1 kind: Job metadata: namespace: default # namespace where the pvc's exist name: volume-migration spec: completions: 1 parallelism: 1 backoffLimit: 3 template: metadata: name: volume-migration labels: name: volume-migration spec: restartPolicy: Never containers: - name: volume-migration image: registry.suse.com/bci/golang:1.24 tty: true command: [ "/bin/sh" ] args: [ "-c", "cp -r -v /mnt/old/. /mnt/new" ] volumeMounts: - name: old-vol mountPath: /mnt/old - name: new-vol mountPath: /mnt/new volumes: - name: old-vol persistentVolumeClaim: claimName: data-source-pvc # change to data source pvc - name: new-vol persistentVolumeClaim: claimName: data-target-pvc # change to data target pvc ================================================ FILE: examples/deployment.yaml ================================================ apiVersion: v1 kind: Service metadata: name: mysql labels: app: mysql spec: ports: - port: 3306 selector: app: mysql clusterIP: None --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: mysql-pvc namespace: default spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 2Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: mysql labels: app: mysql namespace: default spec: selector: matchLabels: app: mysql # has to match .spec.template.metadata.labels strategy: type: Recreate template: metadata: labels: app: mysql spec: restartPolicy: Always containers: - image: mysql:5.6 name: mysql livenessProbe: exec: command: - ls - /var/lib/mysql/lost+found initialDelaySeconds: 5 periodSeconds: 5 ports: - containerPort: 3306 name: mysql volumeMounts: - name: mysql-volume mountPath: /var/lib/mysql env: - name: MYSQL_ROOT_PASSWORD value: "rancher" volumes: - name: mysql-volume persistentVolumeClaim: claimName: mysql-pvc ================================================ FILE: examples/network-policy/backing-image-data-source-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backing-image-data-source namespace: longhorn-system spec: podSelector: matchLabels: longhorn.io/component: backing-image-data-source policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-manager - podSelector: matchLabels: longhorn.io/component: backing-image-data-source ================================================ FILE: examples/network-policy/backing-image-manager-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backing-image-manager namespace: longhorn-system spec: podSelector: matchLabels: longhorn.io/component: backing-image-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-manager ================================================ FILE: examples/network-policy/instance-manager-networking.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: instance-manager namespace: longhorn-system spec: podSelector: matchLabels: longhorn.io/component: instance-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: longhorn.io/component: instance-manager - podSelector: matchLabels: longhorn.io/component: backing-image-data-source ================================================ FILE: examples/network-policy/manager-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-manager namespace: longhorn-system spec: podSelector: matchLabels: app: longhorn-manager policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: longhorn-manager - podSelector: matchLabels: app: longhorn-ui - podSelector: matchLabels: app: longhorn-csi-plugin - podSelector: matchLabels: longhorn.io/managed-by: longhorn-manager matchExpressions: - { key: recurring-job.longhorn.io, operator: Exists } - podSelector: matchExpressions: - { key: longhorn.io/job-task, operator: Exists } - podSelector: matchLabels: app: longhorn-driver-deployer ================================================ FILE: examples/network-policy/recovery-backend-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-recovery-backend namespace: longhorn-system spec: podSelector: matchLabels: longhorn.io/recovery-backend: longhorn-recovery-backend policyTypes: - Ingress ingress: - ports: - protocol: TCP port: 9503 ================================================ FILE: examples/network-policy/ui-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-ui namespace: longhorn-system spec: podSelector: matchLabels: app: longhorn-ui policyTypes: - Ingress ingress: - from: # Depending on the ingress controller setup in your cluster, Change the following # info to allow the traffic from the ingress controller pods to Longhorn UI - namespaceSelector: matchLabels: kubernetes.io/metadata.name: ingress-nginx podSelector: matchLabels: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx ================================================ FILE: examples/network-policy/webhook-network-policy.yaml ================================================ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: longhorn-admission-webhook namespace: longhorn-system spec: podSelector: matchLabels: longhorn.io/admission-webhook: longhorn-admission-webhook policyTypes: - Ingress ingress: - ports: - protocol: TCP port: 9502 ================================================ FILE: examples/pod_with_gev.yaml ================================================ apiVersion: v1 kind: Pod metadata: name: volume-test namespace: default spec: restartPolicy: Always containers: - name: volume-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent livenessProbe: exec: command: - ls - /data/lost+found initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: volv mountPath: /data ports: - containerPort: 80 volumes: - name: volv ephemeral: volumeClaimTemplate: spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 2Gi ================================================ FILE: examples/pod_with_pvc.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-volv-pvc namespace: default spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 2Gi --- apiVersion: v1 kind: Pod metadata: name: volume-test namespace: default spec: restartPolicy: Always containers: - name: volume-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent livenessProbe: exec: command: - ls - /data/lost+found initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: volv mountPath: /data ports: - containerPort: 80 volumes: - name: volv persistentVolumeClaim: claimName: longhorn-volv-pvc ================================================ FILE: examples/restore_to_file.yaml.template ================================================ apiVersion: v1 kind: Pod metadata: name: restore-to-file namespace: longhorn-system spec: nodeName: containers: - name: restore-to-file command: # set restore-to-file arguments here - /bin/sh - -c - longhorn backup restore-to-file '' --output-file '/tmp/restore/' --output-format # the version of longhorn engine should be v0.4.1 or higher image: longhornio/longhorn-engine: imagePullPolicy: IfNotPresent securityContext: privileged: true volumeMounts: - name: disk-directory mountPath: /tmp/restore # the argument should be in this directory env: # set Backup Target Credential Secret here. - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: name: key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: key: AWS_SECRET_ACCESS_KEY - name: AWS_ENDPOINTS valueFrom: secretKeyRef: name: key: AWS_ENDPOINTS volumes: # the output file can be found on this host path - name: disk-directory hostPath: path: /tmp/restore restartPolicy: Never ================================================ FILE: examples/rwx/rwx-nginx-deployment.yaml ================================================ apiVersion: v1 kind: Service metadata: name: rwx-test labels: app: rwx-test spec: ports: - port: 80 selector: app: rwx-test --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: rwx-test namespace: default spec: accessModes: - ReadWriteMany storageClassName: longhorn resources: requests: storage: 1Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: rwx-test labels: app: rwx-test namespace: default spec: replicas: 3 selector: matchLabels: app: rwx-test strategy: type: Recreate template: metadata: labels: app: rwx-test spec: containers: - image: ubuntu:xenial imagePullPolicy: IfNotPresent command: [ "/bin/sh", "-c" ] args: - sleep 10; touch /data/index.html; while true; do date >> /data/index.html; sleep 1; done; name: rwx-test stdin: true tty: true volumeMounts: - mountPath: /data name: rwx-test - image: nginx:stable imagePullPolicy: IfNotPresent name: nginx ports: - containerPort: 80 name: http volumeMounts: - mountPath: /usr/share/nginx/html name: rwx-test restartPolicy: Always volumes: - name: rwx-test persistentVolumeClaim: claimName: rwx-test ================================================ FILE: examples/rwx/storageclass-migratable.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-migratable provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" # 48 hours in minutes fromBackup: "" migratable: "true" ================================================ FILE: examples/simple_pod.yaml ================================================ apiVersion: v1 kind: Pod metadata: name: longhorn-simple-pod namespace: default spec: restartPolicy: Always containers: - name: volume-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent livenessProbe: exec: command: - ls - /data/lost+found initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: volv mountPath: /data ports: - containerPort: 80 volumes: - name: volv persistentVolumeClaim: claimName: longhorn-simple-pvc ================================================ FILE: examples/simple_pvc.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-simple-pvc namespace: default spec: accessModes: - ReadWriteOnce storageClassName: longhorn resources: requests: storage: 1Gi ================================================ FILE: examples/snapshot/existing_backup.yaml ================================================ apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotContent metadata: name: test-existing-backup spec: volumeSnapshotClassName: longhorn driver: driver.longhorn.io deletionPolicy: Delete source: # NOTE: change this to point to an existing backup on the backupstore snapshotHandle: bs://test-vol/backup-625159fb469e492e volumeSnapshotRef: name: test-snapshot-existing-backup namespace: default ================================================ FILE: examples/snapshot/restore_existing_backup.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-existing-backup namespace: default spec: storageClassName: longhorn dataSource: name: test-snapshot-existing-backup kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 2Gi ================================================ FILE: examples/snapshot/restore_pvc_snapshot.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-restore-snapshot-pvc namespace: default spec: storageClassName: longhorn dataSource: name: test-snapshot-pvc kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io accessModes: - ReadWriteOnce resources: requests: storage: 2Gi ================================================ FILE: examples/snapshot/snapshot_existing.yaml ================================================ apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: test-snapshot-existing-backup spec: volumeSnapshotClassName: longhorn source: volumeSnapshotContentName: test-existing-backup ================================================ FILE: examples/snapshot/snapshot_pvc.yaml ================================================ apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: test-snapshot-pvc spec: volumeSnapshotClassName: longhorn source: persistentVolumeClaimName: test-vol ================================================ FILE: examples/snapshot/snapshotclass.yaml ================================================ kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1 metadata: name: longhorn driver: driver.longhorn.io deletionPolicy: Delete #parameters: # csi.storage.k8s.io/snapshotter-secret-name: mysecret # csi.storage.k8s.io/snapshotter-secret-namespace: mysecretnamespace ================================================ FILE: examples/statefulset.yaml ================================================ apiVersion: v1 kind: Service metadata: name: nginx labels: app: nginx spec: ports: - port: 80 name: web selector: app: nginx type: NodePort --- apiVersion: apps/v1 kind: StatefulSet metadata: name: web namespace: default spec: selector: matchLabels: app: nginx # has to match .spec.template.metadata.labels serviceName: "nginx" replicas: 2 # by default is 1 template: metadata: labels: app: nginx # has to match .spec.selector.matchLabels spec: restartPolicy: Always terminationGracePeriodSeconds: 10 containers: - name: nginx image: registry.k8s.io/nginx-slim:0.8 livenessProbe: exec: command: - ls - /usr/share/nginx/html/lost+found initialDelaySeconds: 5 periodSeconds: 5 ports: - containerPort: 80 name: web volumeMounts: - name: www mountPath: /usr/share/nginx/html volumeClaimTemplates: - metadata: name: www spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "longhorn" resources: requests: storage: 1Gi ================================================ FILE: examples/storageclass.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-test provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: Immediate parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" fromBackup: "" fsType: "ext4" # mkfsParams: "-I 256 -b 4096 -O ^metadata_csum,^64bit" # backingImage: "bi-test" # backingImageDataSourceType: "download" # backingImageDataSourceParameters: '{"url": "https://backing-image-example.s3-region.amazonaws.com/test-backing-image"}' # backingImageChecksum: "SHA512 checksum of the backing image" # unmapMarkSnapChainRemoved: "ignored" # diskSelector: "ssd,fast" # nodeSelector: "storage,fast" # recurringJobSelector: '[{"name":"snap-group", "isGroup":true}, # {"name":"backup", "isGroup":false}]' # nfsOptions: "soft,timeo=150,retrans=3" ================================================ FILE: examples/v2/pod_with_pvc.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: longhorn-volv-pvc namespace: default spec: accessModes: - ReadWriteOnce storageClassName: longhorn-v2-data-engine resources: requests: storage: 2Gi --- apiVersion: v1 kind: Pod metadata: name: volume-test namespace: default spec: restartPolicy: Always containers: - name: volume-test image: nginx:stable-alpine imagePullPolicy: IfNotPresent livenessProbe: exec: command: - ls - /data/lost+found initialDelaySeconds: 5 periodSeconds: 5 volumeMounts: - name: volv mountPath: /data ports: - containerPort: 80 volumes: - name: volv persistentVolumeClaim: claimName: longhorn-volv-pvc ================================================ FILE: examples/v2/storageclass.yaml ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: longhorn-v2-data-engine provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: Immediate parameters: numberOfReplicas: "3" staleReplicaTimeout: "2880" fsType: "ext4" dataEngine: "v2" # mkfsParams: "-I 256 -b 4096 -O ^metadata_csum,^64bit" # nodeSelector: "storage,fast" # recurringJobSelector: '[{"name":"snap-group", "isGroup":true}, # {"name":"backup", "isGroup":false}]' # fromBackup: "" # backingImage: "bi-test" # backingImageDataSourceType: "download" # backingImageDataSourceParameters: '{"url": "https://backing-image-example.s3-region.amazonaws.com/test-backing-image"}' # backingImageChecksum: "SHA512 checksum of the backing image" # unmapMarkSnapChainRemoved: "ignored" # diskSelector: "ssd,fast" # nfsOptions: "soft,timeo=150,retrans=3" ================================================ FILE: renovate.json ================================================ { "extends": ["github>longhorn/release:renovate-default"] } ================================================ FILE: scalability/dev/control_plane_grafana_dashboard.json ================================================ { "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 44, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 16, "panels": [], "title": "Workload", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "mode": "continuous-greens" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "id": 7, "interval": "15s", "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum (kube_pod_info{namespace=\"default\"})", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "sum(kube_pod_info{namespace=\"default\"})", "hide": true, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Pod Count", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "x-axis is time in seconds\ny-axis is number of pods", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "id": 4, "interval": "15s", "options": { "bucketOffset": 0, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "kube_pod_status_ready_time{namespace=\"default\"}-kube_pod_created{namespace=\"default\"}", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Histogram of Pod Starting Time", "type": "histogram" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "mode": "continuous-greens" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "id": 2, "interval": "15s", "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by(phase) (kube_pod_status_phase{namespace=\"default\"})", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Pod Count By Phase", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "id": 65, "interval": "15s", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "count (kube_pod_status_ready_time{namespace=\"default\"}-kube_pod_created{namespace=\"default\"} <= 240) OR vector(0)", "hide": false, "legendFormat": "Count", "range": true, "refId": "A" } ], "title": "Number of Running Pod with Starting Time <= 4m", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": true, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "dark-red", "value": -10 }, { "color": "red", "value": 10 } ] } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "mode": "continuous-greens" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "id": 8, "interval": "15s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "deriv (sum (kube_pod_status_phase{namespace=\"default\", phase=\"Running\"})[2m:15s])*60", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Per-minute Rate of Pods Become Running", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 10 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, "id": 10, "interval": "15s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "count (kube_pod_status_ready_time{namespace=\"default\"}-kube_pod_created{namespace=\"default\"} > 240) OR vector(0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Number of Running Pod with Starting Time > 4m", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 6, "interval": "15s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by(node) (kube_pod_info{namespace=\"default\", node!=\"\"})", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Pods per Node", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "id": 12, "interval": "15s", "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "topk (100, kube_pod_status_ready_time{namespace=\"default\"}-kube_pod_created{namespace=\"default\"})", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Pod With Biggest Starting Time (Top 100)", "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "names": [ "pod", "Value" ] } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "id": 14, "interval": "15s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "count(kube_pod_container_status_restarts_total{namespace=\"default\"}>0) OR vector(0)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Crashed Pod Count", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 28, "panels": [], "title": "Longhorn CPU and RAM", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cpu" }, "overrides": [] }, "gridPos": { "h": 18, "w": 24, "x": 0, "y": 42 }, "id": 24, "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{ namespace=\"longhorn-system\"}) by (pod)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CPU Usage Per Pod - Longhorn System", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 16, "w": 24, "x": 0, "y": 60 }, "id": 22, "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"longhorn-system\", container!=\"\", image!=\"\"}) by (pod)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Memory Usage (RSS) Per Pod - Longhorn System", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 16, "w": 24, "x": 0, "y": 76 }, "id": 26, "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",namespace=\"longhorn-system\", container!=\"\", image!=\"\"}) by (pod)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Memory Usage Per Pod - Longhorn System", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 92 }, "id": 58, "panels": [], "title": "ETCD", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 93 }, "id": 30, "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "editorMode": "code", "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", "intervalFactor": 2, "legendFormat": "", "metric": "etcd_server_has_leader", "range": true, "refId": "A", "step": 20 } ], "title": "Up", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 16, "x": 8, "y": 93 }, "id": 56, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (verb) (rate (apiserver_request_total[2m]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "API Server - HTTP Request per Second by Verbs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 101 }, "id": 34, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "editorMode": "code", "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", "intervalFactor": 2, "legendFormat": "Watch Streams", "metric": "grpc_server_handled_total", "range": true, "refId": "A", "step": 4 }, { "datasource": { "uid": "$datasource" }, "editorMode": "code", "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", "intervalFactor": 2, "legendFormat": "Lease Streams", "metric": "grpc_server_handled_total", "range": true, "refId": "B", "step": 4 } ], "title": "Active Streams", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 16, "x": 8, "y": 101 }, "id": 32, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "RPC Rate", "metric": "grpc_server_started_total", "range": true, "refId": "A", "step": 2 }, { "datasource": { "uid": "$datasource" }, "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "RPC Failed Rate", "metric": "grpc_server_handled_total", "refId": "B", "step": 2 } ], "title": "RPC Rate", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 108 }, "id": 36, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}} DB Size", "metric": "", "refId": "A", "step": 4 } ], "title": "DB Size", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 108 }, "id": 38, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", "hide": false, "intervalFactor": 2, "legendFormat": "{{instance}} WAL fsync", "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", "refId": "A", "step": 4 }, { "datasource": { "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", "intervalFactor": 2, "legendFormat": "{{instance}} DB fsync", "metric": "etcd_disk_backend_commit_duration_seconds_bucket", "refId": "B", "step": 4 } ], "title": "Disk Sync Duration", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 108 }, "id": 40, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "process_resident_memory_bytes{job=\"$cluster\"}", "intervalFactor": 2, "legendFormat": "{{instance}} Resident Memory", "metric": "process_resident_memory_bytes", "refId": "A", "step": 4 } ], "title": "Memory", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 115 }, "id": 42, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", "intervalFactor": 2, "legendFormat": "{{instance}} Client Traffic In", "metric": "etcd_network_client_grpc_received_bytes_total", "refId": "A", "step": 4 } ], "title": "Client Traffic In", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 115 }, "id": 44, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", "intervalFactor": 2, "legendFormat": "{{instance}} Client Traffic Out", "metric": "etcd_network_client_grpc_sent_bytes_total", "refId": "A", "step": 4 } ], "title": "Client Traffic Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 115 }, "id": 60, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", "intervalFactor": 2, "legendFormat": "{{instance}} Peer Traffic In", "metric": "etcd_network_peer_received_bytes_total", "range": true, "refId": "A", "step": 4 } ], "title": "Peer Traffic In", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 115 }, "id": 62, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}} Peer Traffic Out", "metric": "etcd_network_peer_sent_bytes_total", "refId": "A", "step": 4 } ], "title": "Peer Traffic Out", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 122 }, "id": 50, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", "intervalFactor": 2, "legendFormat": "Proposal Failure Rate", "metric": "etcd_server_proposals_failed_total", "refId": "A", "step": 2 }, { "datasource": { "uid": "$datasource" }, "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", "intervalFactor": 2, "legendFormat": "Proposal Pending Total", "metric": "etcd_server_proposals_pending", "refId": "B", "step": 2 }, { "datasource": { "uid": "$datasource" }, "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", "intervalFactor": 2, "legendFormat": "Proposal Commit Rate", "metric": "etcd_server_proposals_committed_total", "refId": "C", "step": 2 }, { "datasource": { "uid": "$datasource" }, "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", "intervalFactor": 2, "legendFormat": "Proposal Apply Rate", "refId": "D", "step": 2 } ], "title": "Raft Proposals", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 122 }, "id": 52, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", "intervalFactor": 2, "legendFormat": "{{instance}} Total Leader Elections Per Day", "metric": "etcd_server_leader_changes_seen_total", "refId": "A", "step": 2 } ], "title": "Total Leader Elections Per Day", "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 129 }, "id": 64, "interval": "30s", "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.5", "targets": [ { "datasource": { "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}} Peer round trip time", "metric": "etcd_network_peer_round_trip_time_seconds_bucket", "refId": "A", "step": 2 } ], "title": "Peer round trip time", "type": "timeseries" } ], "refresh": "5m", "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "kube-etcd", "value": "kube-etcd" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "definition": "label_values(etcd_server_has_leader, job)", "hide": 2, "includeAll": false, "label": "cluster", "multi": false, "name": "cluster", "options": [], "query": { "query": "label_values(etcd_server_has_leader, job)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 2, "type": "query" } ] }, "time": { "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Longhorn Control Plane Scalability", "uid": "OUbS8NYIl", "version": 14, "weekStart": "" } ================================================ FILE: scalability/dev/data-plane-grafana_dashboard.json ================================================ { "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.1.5" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], "liveNow": false, "panels": [ { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 4, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 11, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "IOPs", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 2 }, "id": 2, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Random Read IOPs ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 2 }, "id": 6, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Read IOPs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 63, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(volume_name) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Random Read IOPs Per PVC", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 65, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by(volume_name) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Read IOPs Per PVC", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 64, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(pod) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Random Read IOPs Per Pod", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "id": 66, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by(pod) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Read IOPs Per Pod", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, "id": 51, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "average", "range": true, "refId": "A" } ], "title": "Average Random Read IOPs Per Container", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "id": 52, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "average", "range": true, "refId": "A" } ], "title": "Average Sequential Read IOPs Per Container", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, "id": 5, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Random Read IOPs - Number of Reported Containers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, "id": 7, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported containers", "range": true, "refId": "A" } ], "title": "Sequential Read IOPs - Number of Reported Containers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, "id": 10, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "Bandwidth", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "id": 8, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Random Read Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "id": 13, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Sequential Read Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "id": 53, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Random Read Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "id": 54, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Sequential Read Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 59 }, "id": 12, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Random Read Bandwidth - Number of Reported Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 59 }, "id": 14, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Sequential Read Bandwidth - Number of Reported Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 67 }, "id": 15, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "Latency", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, "id": 17, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Random Read Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, "id": 18, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Read Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 76 }, "id": 55, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Random Read Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 76 }, "id": 56, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Sequential Read Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 84 }, "id": 20, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "number of reported containers", "range": true, "refId": "A" } ], "title": "Random Read Latency - Number of Reported Containers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 84 }, "id": 19, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"read\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "number of reported containers", "range": true, "refId": "A" } ], "title": "Sequential Read Latency - Number of Reported Containers", "type": "timeseries" } ], "title": "Read Performance", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 22, "panels": [], "title": "Write Performance", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 24, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "IOPs", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, "id": 26, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Random Write IOPs ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 }, "id": 28, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Write IOPs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "id": 67, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(volume_name) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{volume_name}}", "range": true, "refId": "A" } ], "title": "Random Write IOPs Per PVC", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "id": 69, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(volume_name) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Write IOPs Per PVC", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "id": 68, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(pod) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Random Write IOPs Per Pod", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "id": 70, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, "expr": "sum by(pod) (kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Sequential Write IOPs Per Pod", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "id": 57, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Random Write IOPs Per Container", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow", "value": null }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "id": 58, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Sequential Write IOPs Per Container", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "id": 30, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported containers", "range": true, "refId": "A" } ], "title": "Random Write IOPs - Number of Reported Containers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 32, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_iops{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Sequential Write IOPs - Number of Reported Containers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 41, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "Bandwidth", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, "id": 34, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Random Write Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, "id": 36, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Sequential Write Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "id": 59, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Random Write Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "KiBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "id": 60, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Sequential Write Bandwidth ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 }, "id": 38, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Random Write Bandwidth - Number of Reported Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 }, "id": 40, "interval": "30s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_bandwidth{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Sequential Write Bandwidth - Number of Reported Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 50, "options": { "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "9.1.5", "title": "Latency", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "id": 43, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Random Write Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "id": 45, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"}", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{volume_name}}", "range": true, "refId": "A" } ], "title": "Sequential Write Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, "id": 61, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Random Write Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 60, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "ns" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, "id": 62, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "avg(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], "title": "Average Sequential Write Latency ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 85 }, "id": 47, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"random\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "number of reported pods", "range": true, "refId": "A" } ], "title": "Random Write Latency - Number of Reported Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "light-yellow" }, { "color": "super-light-green", "value": 20 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 85 }, "id": 49, "interval": "30s", "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(kbench_metric_exporter_latency{volume_access_mode=\"$volume_access_mode\", test_mode=\"$test_mode\", rate_limit_type=\"$rate_limit_type\", io_pattern=\"sequential\", io_type=\"write\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "number of reported containers", "range": true, "refId": "A" } ], "title": "Sequential Write Latency - Number of Reported Containers", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "rwx", "value": "rwx" }, "description": "", "hide": 0, "includeAll": false, "label": "Volume Access Mode", "multi": false, "name": "volume_access_mode", "options": [ { "selected": false, "text": "rwo", "value": "rwo" }, { "selected": true, "text": "rwx", "value": "rwx" } ], "query": "rwo,rwx", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "current": { "selected": false, "text": "write-only", "value": "write-only" }, "description": "Test Mode", "hide": 0, "includeAll": false, "multi": false, "name": "test_mode", "options": [ { "selected": false, "text": "read-only", "value": "read-only" }, { "selected": true, "text": "write-only", "value": "write-only" }, { "selected": false, "text": "read-write", "value": "read-write" } ], "query": "read-only,write-only,read-write", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "current": { "selected": false, "text": "rate-limit", "value": "rate-limit" }, "description": "Rate Limit Type", "hide": 0, "includeAll": false, "multi": false, "name": "rate_limit_type", "options": [ { "selected": false, "text": "no-rate-limit", "value": "no-rate-limit" }, { "selected": true, "text": "rate-limit", "value": "rate-limit" } ], "query": "no-rate-limit,rate-limit", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "utc", "title": "Reference Setup, Scalability, Performance, and Sizing", "uid": "2fqV-mhSz", "version": 21, "weekStart": "" } ================================================ FILE: scalability/reference-setup-performance-scalability-and-sizing-guidelines/README.md ================================================ # Reference Setup, Performance, Scalability, and Sizing Guidelines In this document, we present the reference setup, performance, scalability, and sizing guidelines when using the Longhorn storage system. In practice, users deploy Longhorn in a vast array of different cluster specifications, making it impossible for us to test all potential setups. Therefore, we will select and test some typical environments for users' reference. By providing these references, users can gain insight into how Longhorn would perform in a similar cluster specification. In summary, in the reports, we will perform various tests against Longhorn data plane and control plane to explore the Longhorn capability. We adjust the parameter like: number of workload pods, IO pattern, number of Longhorn volumes, number of nodes in the cluster, the size of Longhorn volumes, ... Then we measure the Longhorn data plane performance (read/write performance) and Longhorn control plane performance (speed of volume provisioning/attaching/detaching/mounting/unmounting) We are building the reports for 2 type of environments: `public cloud` and `on-prem`. The reason that we want to separate `public cloud` and `on-prem` is that they are 2 popular usecases with typically different configurations. For example, `public cloud` usually does not offer fast disk and network for small/medium instances while this configuration is possible in the on-prem environment. ## Public Cloud 1. [Medium Node Spec](./public-cloud/medium-node-spec.md) 1. [Big Node Spec](./public-cloud/big-node-spec.md) (targeted for `v1.7.0`) ## On-Prem 1. [Medium Node Spec](./on-prem/medium-node-spec.md) (targeted for `v1.7.0`) 1. [Big Node Spec](./on-prem/big-node-spec.md) (targeted for `v1.7.0`) ================================================ FILE: scalability/reference-setup-performance-scalability-and-sizing-guidelines/on-prem/big-node-spec.md ================================================ TODO ================================================ FILE: scalability/reference-setup-performance-scalability-and-sizing-guidelines/on-prem/medium-node-spec.md ================================================ TODO ================================================ FILE: scalability/reference-setup-performance-scalability-and-sizing-guidelines/public-cloud/big-node-spec.md ================================================ TODO ================================================ FILE: scalability/reference-setup-performance-scalability-and-sizing-guidelines/public-cloud/medium-node-spec.md ================================================ # Reference Setup, Performance, Scalability, and Sizing Guidelines: Public Cloud - Medium Node Spec ## Table of contents 1. [Cluster Spec](#cluster-spec) 1. [Node Spec](#node-spec) 1. [Network Spec](#network-spec) 1. [Disk Spec](#disk-spec) 1. [Kubernetes Spec](#kubernetes-spec) 1. [Longhorn Config](#longhorn-config) 1. [Additional Components](#additional-components) 1. [Workload design](#workload-design) 1. [Read and Write Tests Design](#read-and-write-performance) 1. [Longhorn Control Tests Design](#longhorn-control-plane-performance) 1. [Maximum Volume Size Tests Design](#maximum-volume-size-tests-design) 1. [Load Balancing](#load-balancing) 1. [Read Performance](#read-performance) 1. [Random Read IOPs - Stress Tests](#random-read-iops---stress-tests) 1. [Random Read IOPs - Rate Limited](#random-read-iops---rate-limited) 1. [Sequential Read IOPs - Stress Tests](#sequential-read-iops---stress-tests) 1. [Sequential Read IOPs - Rate Limited](#sequential-read-iops---rate-limited) 1. [Random Read Bandwidth - Stress Tests](#random-read-bandwidth---stress-tests) 1. [Random Read Bandwidth - Rate Limited](#random-read-bandwidth---rate-limited) 1. [Sequential Read Bandwidth - Stress Tests](#sequential-read-bandwidth---stress-tests) 1. [Sequential Read Bandwidth - Rate Limited](#sequential-read-bandwidth---rate-limited) 1. [Random Read Latency - Stress Tests](#random-read-latency---stress-tests) 1. [Sequential Read Latency - Stress Tests](#sequential-read-latency---stress-tests) 1. [Write Performance](#write-performance) 1. [Random Write IOPs - Stress Tests](#random-write-iops---stress-tests) 1. [Random Write IOPs - Rate Limited](#random-write-iops---rate-limited) 1. [Sequential Write IOPs - Stress Tests](#sequential-write-iops---stress-tests) 1. [Sequential Write IOPs - Rate Limited](#sequential-write-iops---rate-limited) 1. [Random Write Bandwidth - Stress Tests](#random-write-bandwidth---stress-tests) 1. [Random Write Bandwidth - Rate Limited](#random-write-bandwidth---rate-limited) 1. [Sequential Write Bandwidth - Stress Tests](#sequential-write-bandwidth---stress-tests) 1. [Sequential Write Bandwidth - Rate Limited](#sequential-write-bandwidth---rate-limited) 1. [Random Write Latency - Stress Tests](#random-write-latency---stress-tests) 1. [Sequential Write Latency - Stress Tests](#sequential-write-latency---stress-tests) 1. [Longhorn Control Plane Performance](#longhorn-control-plane-performance) 1. [Volume Maximum Size](#volume-maximum-size) 1. [FAQs](#faqs) ## Cluster Spec ### Node Spec * EC2 instance type: ec2 m5zn.2xlarge * 8vCPUs, 32GB RAM * Root disk * Size 50GB * Type EBS gp3 * OS: Ubuntu 22.04.3 LTS (Jammy Jellyfish) * Kernel version: 6.2.0-1017-aws **Comment:** We choose m5zn.2xlarge ec2 instance because it has consistent EBS performance unlike other similar (in terms of CPU and memory) instances which have baseline and burst EBS performance. Consistent EBS performance is important because we are using dedicated EBS disk for Longhorn storage. If the EBS performance is inconsistent, we will see inconsistent test result over time. See more at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html . ### Network Spec * Network bandwidth: * Baseline bandwidth 10 Gbps * Burst bandwidth: Up to 25 Gbps in short time * Network latency: ~0.3ms RTT via ping command ### Disk Spec * We are using dedicated disk for Longhorn volumes' replicas on the nodes. We select a typical EBS volume with average IOPs and bandwidth performance: * Single EC2 EBS gp3 * 1TB * IOPs set to 10000 * Throughput set to 360MiB/s * Formatted as ext4 filesystem **Comment:** * We choose 10000 for EBS disk's IOPs simply because it is a middle value between minimum value 3000 and maximum value 16000 of the gp3 EBS disk * We choose 360MiB/s for EBS disk's bandwidth because the m5zn.2xlarge EC2 instance has EBS bandwidth 396.25 MiB/s. If we choose a bigger value than 396.25 MiB/s for EBS disk's bandwidth, the ec2 instance would not be able to push EBS disk to that value ### Kubernetes Spec * Kubernetes Version: v1.27.10+rke2r1 * CNI plugin: Calico * Control plane nodes are separated from worker nodes ### Longhorn Config * Longhorn version: v1.6.0 * Settings: * Using dedicated disk for Longhorn instead of root disk * The number of replicas per volume is 3 * Storage Minimal Available Percentage setting: 10% * As we are using dedicated disk, we don't need big reserve storage as mentioned in best practice https://longhorn.io/docs/1.6.0/best-practices/#minimal-available-storage-and-over-provisioning * Storage Over Provisioning Percentage setting: 110% * We are planning to fill 15GB for each 20GB volume If we schedule maximum amount, it would be 1100 GiB and actual usage will be (15/20)*1200 = 825GiB. This leaves 100GiB as 10% Storage Minimal Available Percentage setting plus some volumes' filesystem space overhead ### Additional Components * We deployed [Rancher monitoring](https://ranchermanager.docs.rancher.com/integrations-in-rancher/monitoring-and-alerting) which is a downstream version of [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack). Note that this monitoring system generally consumes a certain amount of CPU, memory, and disk space on the node. We are deploying it with: * CPU request: 750m * CPU limit: 1000m * Memory request: 750Mi * Memory limit: 3000Mi * Data retention size: 25GiB * We deployed [Local Path Provisioner](https://github.com/rancher/local-path-provisioner) to test baseline storage performance when using local storage in each node directly. ## Workload design ### Read and Write Tests Design We use [Kbench](https://github.com/yasker/kbench) which is a tool to test Kubernetes storage. The idea of Kbench is that it deploys a pod with a Longhorn volume. Then it runs various `fio` jobs (specified by user) to test multiple performance aspects of the volume (IOPs, bandwidth, latency). The pod runs the test repeatedly and exposes the result as Prometheus metrics. We then collect and visualize the data using Grafana. Traditionally, kbench only deploys 1 single pod to test 1 volume. However, in this report, we will gradually scale up the number of Kbench pods. Since each pod stress-tests its Longhorn volume, as the number of Kbench pods go up, we can simulate the situation in which the cluster has many pods doing IO aggressively. From there we can see the performance characteristic of Longhorn storage system as there are more and more IO intensive workloads. We also perform the tests in which Kbench pods are rate-limited. From the rate-limited tests, we can further verify the result from the stress tests. To avoid cache effect, we choose the `fio` test size to be big enough. See the more details at https://github.com/longhorn/kbench#:~:text=the%20YAML%20locally.-,As%20mentioned%20above%2C%20for%20formal%20benchmark%2C%20the%20size%20should%20be%20at%20least%2025%20times%20the%20read/write%20bandwidth%20to%20avoid%20the%20caching%20impacting%20the%20result.,-Step%20to%20deploy ### Longhorn Control Plane Tests Design The read and write performance tests above mostly concern the Longhorn data plane. In order to discover the scalability of Longhorn control plane, we also run tests that have non-IO workloads. These tests will help us to answer questions such as: 1. Maximum number of Longhorn volumes inside the cluster 1. Maximum number of Longhorn volumes attached to a node 1. Maximum number of Longhorn replicas on a node ### Maximum Volume Size Tests Design From the experience and code analysis, the maximum Longhorn volume size is limited by the replica rebuilding time. In this report, we will measure the replica rebuilding time as the size of Longhorn volume is getting bigger. From there user can estimate the maximum volume size they can set given the information about how Longhorn it would take if Longhorn need to rebuild replicas for that volume. ### Load Balancing In this report, we will distribute the load evenly across worker nodes: 1. Each worker nodes will have relative similar number of Kbench pods 2. Each worker nodes will have relative similar number of Longhorn replicas This setup optimize efficiency of the cluster and Longhorn system. In practice, this balance is also what users usually strive for. ## Read Performance ### Random Read IOPs - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-read-iops`: specifies that we are running [random read iops job](https://github.com/longhorn/kbench/blob/main/fio/bandwidth-random-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 10000 > * Longhorn: 22180 drawing **Analysis & Conclusion**: * Because Longhorn has 3 replicas, it can read from 3 replicas concurrently thus may produce better read performance Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random read IOPs is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 10639 random read IOPs on its Longhorn volume > * Total random IOPs can be achieved by all 3 Longhorn volumes is 31917 drawing **Analysis & Conclusion**: * Since each EBS volume on the host is provisioned with 10000 IOPs. Total IOPs of 3 EBS volumes is around 30000 * It looks like Longhorn system is able to reach the maximum IOPs capacity of the 3 host EBS volumes. ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average random read IOPs per Longhorn volume is 5320. Total random IOPs is 31920 > * At 9 pods, the average random read IOPs per Longhorn volume is 3541. Total random IOPs is 31869 > * At 12 pods, the average random read IOPs per Longhorn volume is 2633. Total random IOPs is 31596 > * At 15 pods, the average random read IOPs per Longhorn volume is 2116. Total random IOPs is 31740 drawing Note: the areas with the while background in the graph are transition periods. We can ignore the data in these areas. **Analysis & Conclusion**: * From the scaling test so far, we can see that the total random read IOPs of all Longhorn volumes remains relatively the same (around 31500) when the number of Kbench pods increase. If we call the average random read IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 31500. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 31500 random read IOPs * If each workload pod is doing 1000 random IOPs on average, there can be about 31 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 31500) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth #### 1 control plane node + 6 worker nodes We double the number worker nodes (from 3 to 6) and double the number of Kbench pods (from 15 to 30) **Result**: > * The average random read IOPs per Longhorn volume is the same 2127 > * The total random IOPs can be achieved by all Longhorn volumes is doubled 63810 drawing **Analysis & Conclusion**: * Since the load is evenly distributed, we can see a linear relationship between total random IOPs and number of nodes: when the number of nodes is doubled, total random IOPs is doubled * From this reference, users can estimate how many worker nodes with the specified spec they need to achieve their target total random read IOPs ### Random Read IOPs - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 617 IOPs and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pod to 51 to see if the system is able to achieve 31467 random read IOPs.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE_IOPS value: "617" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The average random read IOPs per Longhorn volume is 616 > * The total random IOPs can be achieved by all 51 Longhorn volumes is 31416 drawing **Comments:** * As discussed in the [Random Read IOPs - Stress Tests](#random-read-iops---stress-tests) section above, we come up with the conclusion that `If we call the average random read IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 31500` * The test result here further confirm this conclusion as x * y = 616*51 = 31416 (slightly lower than 31467 but it is insignificant difference) ### Sequential Read IOPs - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE_IOPS # value: "617" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-read-iops`: specifies that we are running [sequential read iops job](https://github.com/longhorn/kbench/blob/main/fio/iops-sequential-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 10000 > * Longhorn: 39772 drawing **Analysis & Conclusion**: * We observe Longhorn volume achieves much bigger sequential read IOPs compares to the local path volume. This can be explained by 2 factors: * Each Longhorn volume has 3 replicas so it can read concurrently from multiple replicas (each replica is on a different EBS disk) while the local path volume is reading from single EBS disk * When running Kbench with Longhorn volume, there is IO merging at both the kernel layer and EBS layer. The IO merging can cause multiple Kbench IOs to be submitted as 1 therefore increasing the IOPs. This IO merging effect doesn't happen when running Kbench with local path volume. See more detailed explanation at https://github.com/longhorn/longhorn/pull/7905#issuecomment-1945463272 Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential read IOPs is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 19840 sequential read IOPs on its Longhorn volume > * Total sequential IOPs can be achieved by all 3 Longhorn volumes is: 59520 drawing ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average sequential read IOPs per Longhorn volume is 10132. Total sequential read IOPs is 60792 > * At 9 pods, the average sequential read IOPs per Longhorn volume is 6875. Total sequential read IOPs is 61875 > * At 12 pods, the average sequential read IOPs per Longhorn volume is 5210. Total sequential read IOPs is 62520 > * At 15 pods, the average sequential read IOPs per Longhorn volume is 4159. Total sequential read IOPs is 62385 drawing Note: the areas with the while background in the graph are transition periods. We can ignore the data in these areas. **Analysis & Conclusion**: * From the scaling test so far, we can see that the total sequential read IOPs of all Longhorn volumes remain relative same around 61500 when the number of Kbench pods increase. If we call the average sequential read IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 61500. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 61500 sequential read IOPs * If each workload pod is doing 1205 sequential IOPs on average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 61500) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be reduced) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth #### 1 control plane node + 6 worker nodes We double the number worker nodes (from 3 to 6) and double the number of Kbench pods (from 15 to 30) **Result**: > * The average sequential read IOPs per Longhorn volume is relatively the same around 4150 > * The total sequential IOPs can be achieved by all Longhorn volumes is doubled to 124500 drawing **Analysis & Conclusion**: * Since the load is evenly distributed, we can see a linear relationship between total sequential IOPs and number of nodes: when the number of nodes is doubled, total sequential IOPs is doubled * From this reference, users can estimate how many worker nodes with the specified spec they need to achieve their target total sequential read IOPs ### Sequential Read IOPs - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 1205 sequential read IOPs and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pods to 51 to see if the system is able to achieve 61455 sequential read IOPs.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE_IOPS value: "1205" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The sequential read IOPs per Longhorn volume is 1190 > * The total sequential read IOPs can be achieved by all 51 Longhorn volumes is 60690 drawing **Comments:** * As discussed in the [Sequential Read IOPs - Stress Tests](#sequential-read-iops---stress-tests) section above, we come up with the conclusion that `If we call the sequential read IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 61500.` * The test result here further confirm this conclusion as x * y = 1190*51 = 60690 (slightly lower than 61500 but it is insignificant difference) ### Random Read Bandwidth - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE_IOPS # value: "1000" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE_IOPS # value: "1000" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-read-bandwidth`: specifies that we are running [random read bandwidth job](https://github.com/longhorn/kbench/blob/main/fio/bandwidth-random-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 362 MiB/s > * Longhorn: 874 MiB/s drawing **Analysis & Conclusion**: * Because Longhorn has 3 replicas, it can read from 3 replicas concurrently, and thus may produce better read performance Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random read bandwidth is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 386 MiB/s random read bandwidth on its Longhorn volume > * Total random read bandwidth can be achieved by all 3 Longhorn volumes is 1158 drawing **Analysis & Conclusion**: * Since each EBS volume on the host is provisioned with 360 MiB/s. Total IOPs of 3 EBS volumes is around 1080 MiB/s It looks like Longhorn system is able to reach the maximum IOPs capacity of the 3 host EBS volumes. Longhorn is able to achieve slightly above 1080 MiB/s maybe due to small number of IO mergings or EBS actual bandwidth might be a bit higher the specified limit. ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average random read bandwidth per Longhorn volume is 196 MiB/s. Total random bandwidth is 1176 MiB/s > * At 9 pods, the average random read bandwidth per Longhorn volume is 131 MiB/s. Total random bandwidth is 1179 MiB/s > * At 12 pods, the average random read bandwidth per Longhorn volume is 97.5 MiB/s. Total random bandwidth is 1170 MiB/s > * At 15 pods, the average random read bandwidth per Longhorn volume is 77.5 MiB/s. Total random bandwidth is 1162 MiB/s drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total random read bandwidth of all Longhorn volumes remain relatively the same around 1160 MiB/s when the number of Kbench pods increase. If we call the average random read bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 1160. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 1160 MiB/s random read bandwidth * If each workload pod is doing 22.7 MiB/s random read bandwidth on average, there can be about 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 1160) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth #### 1 control plane node + 6 worker nodes We double the number worker nodes (from 3 to 6) and double the number of Kbench pods (from 15 to 30) **Result**: > * The average random read bandwidth per Longhorn volume is the same around 77.3 MiB/s > * The total random bandwidth can be achieved by all Longhorn volumes is doubled 2319 MiB/s drawing **Analysis & Conclusion**: * Since the load is evenly distributed, we can see a linear relationship between total random read bandwidth and number of nodes: when the number of nodes is doubled, total random read bandwidth is doubled * From this reference, users can estimate how many worker nodes with the specified spec they need to achieve their target total random read bandwidth. ### Random Read Bandwidth - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 22.7 MiB/s random read bandwidth and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pods to 51 to see if the system is able to achieve 1157 MiB/s random read bandwidth
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE value: "23244k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The random read bandwidth per Longhorn volume is 22.3 MiB/s > * The total random read bandwidth can be achieved by all 51 Longhorn volumes is 1137 MiB/s drawing **Analysis & Conclusion**: * As discussed in the [Random Read Bandwidth - Stress Tests](#random-read-bandwidth---stress-tests) section above, we come up with the conclusion that `If we call the random read bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 1160.` * The test result here further confirm this conclusion as x * y = 22.3*51 = 1137 (slightly lower than 1160 as the CPU contention and other factors kick in) ### Sequential Read Bandwidth - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "23244k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "23244k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-read-bandwidth`: specifies that we are running [sequential read bandwidth job](https://github.com/longhorn/kbench/blob/main/fio/bandwidth-sequential-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 362 MiB/s > * Longhorn: 669 MiB/s drawing **Analysis & Conclusion**: * Because Longhorn has 3 replicas, it can read from 3 replicas concurrently thus may produce better read performance * _Note that the sequential read bandwidth is smaller than the random read bandwidth when testing against a single Longhorn volume of 3 replicas. We observed that this behavior happen when there is IO merging happen at kernel level and when the Longhorn volume has multiple replicas on different nodes. We don't understand the root cause of this behavior yet so we created a GitHub ticket to investigate it later https://github.com/longhorn/longhorn/issues/8108_ Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential read bandwidth is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 378 MiB/s sequential read bandwidth on its Longhorn volume > * Total sequential read bandwidth can be achieved by all 3 Longhorn volumes is 1134 drawing **Analysis & Conclusion**: * Since each EBS volume on the host is provisioned with 360 MiB/s. Total IOPs of 3 EBS volumes is around 1080 MiB/s It looks like Longhorn system is able to reach the maximum bandwidth capacity of the 3 host EBS volumes. Longhorn is able to achieve slightly above 1080 MiB/s maybe due to small number of IO mergings or EBS actual bandwidth might be a bit higher the specified limit ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average sequential read bandwidth per Longhorn volume is 192 MiB/s. Total sequential read bandwidth is 1152 MiB/s > * At 9 pods, the average sequential read bandwidth per Longhorn volume is 126 MiB/s. Total sequential read bandwidth is 1179 MiB/s > * At 12 pods, the average sequential read bandwidth per Longhorn volume is 94 MiB/s. Total sequential read bandwidth is 1128 MiB/s > * At 15 pods, the average sequential read bandwidth per Longhorn volume is 74 MiB/s. Total sequential read bandwidth is 1110 MiB/s drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total sequential read bandwidth of all Longhorn volumes remain relatively the same around 1110 MiB/s when the number of Kbench pods increase. If we call the average sequential read bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 1110 Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 1110 MiB/s sequential read bandwidth * If each workload pod is doing 21.7 MiB/s sequential read bandwidth on average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 1160) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth #### 1 control plane node + 6 worker nodes We double the number worker nodes (from 3 to 6) and double the number of Kbench pods (from 15 to 30) **Result**: > * The average sequential read bandwidth per Longhorn volume is the same around 74 MiB/s > * The total sequential read bandwidth can be achieved by all Longhorn volumes is doubled 2220 MiB/s drawing **Analysis & Conclusion**: * Since the load is evenly distributed, we can see a linear relationship between total sequential read bandwidth and number of nodes: when the number of nodes is doubled, total sequential read bandwidth is doubled * From this reference, users can estimate how many worker nodes with the specified spec they need to achieve their target total sequential read bandwidth ### Sequential Read Bandwidth - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 21.7 MiB/s sequential read bandwidth and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pods to 51 to see if the system is able to achieve 1107 MiB/s sequential read bandwidth.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE value: "22220k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The sequential read bandwidth per Longhorn volume is 21.7 MiB/s > * The total sequential read bandwidth can be achieved by all 51 Longhorn volumes is 1107 MiB/s drawing **Analysis & Conclusion**: * As discussed in the [Sequential Read Bandwidth - Stress Tests](#sequential-read-bandwidth---stress-tests) section above, we come up with the conclusion that `If we call the sequential read bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 1110` * The test result here further confirm this conclusion as x * y = xxx*51 = 1107 (relatively same as 1110) ### Random Read Latency - Stress Tests In this test, we use a cluster that has 1 control plane node + 3 worker nodes. First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-read-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-read-latency`: specifies that we are running [random read latency job](https://github.com/longhorn/kbench/blob/main/fio/latency-random-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 500 microsecond > * Longhorn: 750 microsecond drawing **Analysis & Conclusion**: * Because the IO path of a Longhorn volume is more complicated than local path provisioner, it is expected that the Longhorn volume has bigger random read latency Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random read latency is affected when there are more and more pods. **Result**: | Number of Kbench pods | Average Random Read Latency (ns) | |-----------------------|----------------------------------| | 1 | 742955 | | 3 | 750170 | | 6 | 774555 | | 9 | 790714 | | 12 | 810890 | | 15 | 832421 | | 18 | 858495 | | 21 | 876277 | | 24 | 902481 | | 27 | 934727 | | 30 | 972684 | | 33 | 1014888 | | 36 | 1117489 | | 39 | 1212721 | | 42 | 1304671 | | 45 | 1391987 | | 48 | 1488104 | | 51 | 1581291 | drawing **Analysis & Conclusion**: * As the number of kbench pods increase, the average random read latency of each Longhorn volume increase in non-linear fashion. ### Sequential Read Latency - Stress Tests In this test, we use a cluster that has 1 control plane node + 3 worker nodes. First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-read-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: read-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-read-latency`: specifies that we are running [sequential read latency job](https://github.com/longhorn/kbench/blob/main/fio/latency-sequential-read.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 480 microsecond > * Longhorn: 740 microsecond drawing **Analysis & Conclusion**: * Because the IO path of a Longhorn volume is longer than local path provisioner, it is expected that the Longhorn volume has bigger sequential read latency Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential read latency is affected when there are more and more pods. **Result**: | Number of Kbench pods | Average Random Read Latency (ns) | |-----------------------|----------------------------------| | 1 | 736699 | | 3 | 745284 | | 6 | 766109 | | 9 | 790306 | | 12 | 808654 | | 15 | 827570 | | 18 | 847532 | | 21 | 869385 | | 24 | 898042 | | 27 | 926866 | | 30 | 959980 | | 33 | 1029409 | | 36 | 1122368 | | 39 | 1211142 | | 42 | 1306095 | | 45 | 1397748 | | 48 | 1493102 | | 51 | 1582556 | drawing **Analysis & Conclusion**: * Sequential read latency result is similar to random read latency * As the number of kbench pods increase, the average sequential read latency of each Longhorn volume increase in non-linear fashion ## Write Performance ### Random Write IOPs - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### > 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-write-iops`: specifies that we are running [random write iops job](https://github.com/longhorn/kbench/blob/main/fio/iops-random-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 9980 > * Longhorn: 1150 drawing **Analysis & Conclusion**: * Longhorn write speed is slower than local path volume because of several factors: * Because Longhorn has 3 replicas, it has to write to 3 replicas for each IO * By default, Longhorn has revision counter enabled which means that for each write IO, Longhorn has to do an additional write IO to write the revision counter * The Longhorn v1 tgt-iSCSI stack introduces some overhead Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random write IOPs is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 1180 random write IOPs on its Longhorn volume > * Total random write IOPs can be achieved by all 3 Longhorn volumes is 3540 drawing ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average random write IOPs per Longhorn volume is 852. Total random write IOPs is 5112 > * At 9 pods, the average random write IOPs per Longhorn volume is 576. Total random write IOPs is 5184 > * At 12 pods, the average random write IOPs per Longhorn volume is 436. Total random write IOPs is 5232 > * At 15 pods, the average random write IOPs per Longhorn volume is 350. Total random write IOPs is 5250 drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total random write IOPs of all Longhorn volumes remain relatively the same around 5250 when the number of Kbench pods increase * Since each EBS volume on the host is provisioned with 10000 IOPs. It looks like Longhorn system is able to reach half of the IOPs capacity of the host EBS volumes. This is due to the revision counter overhead. * If we call the average random write IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 5250. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 5250 random write IOPs * If each workload pod is doing 102 random write IOPs in average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 5250) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth #### 1 control plane node + 6 worker nodes We double the number worker nodes (from 3 to 6) and double the number of Kbench pods (from 15 to 30) **Result**: > * The average random write IOPs per Longhorn volume is the same around 350 > * The total random write IOPs can be achieved by all Longhorn volumes is doubled 10500 drawing **Analysis & Conclusion**: * Since the load is evenly distributed, we can see a linear relationship between total random write IOPs and number of nodes: when the number of nodes is doubled, total random write IOPs is doubled * From this reference, users can estimate how many worker nodes with the specified spec they need to achieve their target total random write IOPs ### Random Write IOPs - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 102 random write IOPs and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pod to 51 to see if the system is able to achieve 102 * 51=5202 random write IOPs.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE_IOPS value: "102" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 5 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The average random write IOPs per Longhorn volume is 102 > * The total random write IOPs can be achieved by all 51 Longhorn volumes is 5202 drawing **Analysis & Conclusion**: * As discussed in the [Random Write IOPs - Stress Tests](#random-write-iops---stress-tests) section above, we come up with the conclusion that `If we call the average random write IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 5250` * The test result here further confirm this conclusion as 102 * 51 = 5202 ### Sequential Write IOPs - Stress Tests We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 30 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-write-iops`: specifies that we are running [sequential write iops job](https://github.com/longhorn/kbench/blob/main/fio/iops-sequential-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 10100 > * Longhorn: 2340 drawing **Analysis & Conclusion**: * Longhorn write speed is slower than local path volume because of several factors: * Because Longhorn has 3 replicas, it has to write to 3 replicas for each IO * By default, Longhorn has revision counter enabled which means that for each write IO, Longhorn has to do an additional write IO to write the revision counter * The Longhorn v1 tgt-iSCSI stack introduces some overhead Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential write IOPs is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 2350 sequential write IOPs on its Longhorn volume > * Total sequential write IOPs can be achieved by all 3 Longhorn volumes is 7050 drawing ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average sequential write IOPs per Longhorn volume is 1745. Total sequential write IOPs is 10470 > * At 9 pods, the average sequential write IOPs per Longhorn volume is 1165. Total sequential write IOPs is 10485 > * At 12 pods, the average sequential write IOPs per Longhorn volume is 871. Total sequential write IOPs is 10452 > * At 15 pods, the average sequential write IOPs per Longhorn volume is 695. Total sequential write IOPs is 10425 drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total sequential write IOPs of all Longhorn volumes remain relatively the same around 10400 when the number of Kbench pods increase * Since each EBS volume on the host is provisioned with 10000 IOPs. It looks like Longhorn system is able to reach the IOPs capacity of the host EBS volumes. Due to the IO merge, Longhorn does not need to update the revision counter frequently. Then the performance degradation caused by the revision counter is greatly reduced. * If we call the average sequential write IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 10400. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 10400 sequential write IOPs * If each workload pod is doing 203 sequential write IOPs on average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 10400) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth ### Sequential Write IOPs - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 203 sequential write IOPs and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pod to 51 to see if the system is able to achieve 203 * 51=10353 sequential write IOPs.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-iops" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE_IOPS value: "203" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 5 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The average sequential write IOPs per Longhorn volume is 201 > * The total sequential write IOPs can be achieved by all 51 Longhorn volumes is 10251 drawing **Analysis & Conclusion**: * As discussed in the [Sequential Write IOPs - Stress Tests](#sequential-write-iops---stress-tests) section above, we come up with the conclusion that `If we call the average sequential write IOPs each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 10400` * The test result here further confirm this conclusion as 201 * 51 = 10251 (slightly lower than 10400) ### Random Write Bandwidth - Stress Tests #### 1 control plane node + 3 worker nodes We start with the cluster that has 1 control plane node + 3 worker nodes. ##### > 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "22220k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "22220k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-write-bandwidth`: specifies that we are running [random write bandwidth job](https://github.com/longhorn/kbench/blob/main/fio/bandwidth-random-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 362 MiB/s > * Longhorn: 159 MiB/s drawing **Analysis & Conclusion**: * Longhorn write speed is slower than local path volume because of several factors: * Because Longhorn has 3 replicas, it has to write to 3 replicas for each IO * By default, Longhorn has revision counter enabled which means that for each write IO, Longhorn has to do an additional write IO to write the revision counter * The Longhorn v1 tgt-iSCSI stack introduces some overhead Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random write bandwidth is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 118 MiB/s random write bandwidth on its Longhorn volume > * Total random write bandwidth can be achieved by all 3 Longhorn volumes is 354 MiB/s drawing **Analysis & Conclusion**: * Since each EBS volume on the host is provisioned with 360 MiB/s. It looks like Longhorn system is able to reach the maximum bandwidth capacity of the host EBS disks ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average random write bandwidth per Longhorn volume is 63.2 MiB/s. Total random write bandwidth is 379.2 MiB/s > * At 9 pods, the average random write bandwidth per Longhorn volume is 42.2 MiB/s. Total random write bandwidth is 379.8 MiB/s > * At 12 pods, the average random write bandwidth per Longhorn volume is 31.4 MiB/s. Total random write bandwidth is 376.8 MiB/s > * At 15 pods, the average random write bandwidth per Longhorn volume is 25.1 MiB/s. Total random write bandwidth is 376.5 MiB/s drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total random write bandwidth of all Longhorn volumes remain relatively the same around 376.5 MiB/s when the number of Kbench pods increase If we call the average random write bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 376.5. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 376.5 MiB/s random write bandwidth * If each workload pod is doing 7.38 MiB/s random read bandwidth on average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 376.5) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth ### Random Write Bandwidth - Rate Limited Tests In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 7.38 MiB/s random write bandwidth and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pods to 51 to see if the system is able to achieve 376.38 MiB/s random write bandwidth.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE value: ",7559k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The random write bandwidth per Longhorn volume is 7.38 MiB/s > * The total random write bandwidth can be achieved by all 51 Longhorn volumes is 376.38 MiB/s drawing **Analysis & Conclusion**: * As discussed in the [Random Write Bandwidth - Stress Tests](#random-write-bandwidth---stress-tests) section above, we come up with the conclusion that `If we call the random write bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 376.5` * The test result here further confirm this conclusion as x * y = 7.38*51 = 376.38 ### Sequential Write Bandwidth - Stress Tests We start with the cluster that has 1 control plane node + 3 worker nodes. ##### 1 Workload Pod Scale First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "22220k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: "22220k," volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-write-bandwidth`: specifies that we are running [sequential write bandwidth job](https://github.com/longhorn/kbench/blob/main/fio/bandwidth-sequential-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 362 MiB/s > * Longhorn: 291 MiB/s drawing **Analysis & Conclusion**: * Longhorn write speed is slower than local path volume because of several factors: * Because Longhorn has 3 replicas, it has to write to 3 replicas for each IO * By default, Longhorn has revision counter enabled which means that for each write IO, Longhorn has to do an additional write IO to write the revision counter * The Longhorn v1 tgt-iSCSI stack introduces some overhead Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential write bandwidth is affected when there are more and more IO intensive pods. ##### 3 Workload Pods Scale Scaling workload from 1 to 3 pods. **Result**: > * Each Kbench pod is able to achieve 127 MiB/s sequential write bandwidth on its Longhorn volume > * Total sequential write bandwidth can be achieved by all 3 Longhorn volumes is 381 MiB/s drawing **Analysis & Conclusion**: * Since each EBS volume on the host is provisioned with 360 MiB/s It looks like Longhorn system is able to reach the maximum bandwidth capacity of the host EBS disks ##### > 3 Workload Pods Scale Scaling workload from 3 to 6, then 6 to 9, then 9 to 12, then 12 to 15 **Result**: > * At 6 pods, the average sequential write bandwidth per Longhorn volume is 63.3 MiB/s. Total sequential write bandwidth is 379.8 MiB/s > * At 9 pods, the average sequential write bandwidth per Longhorn volume is 42.6 MiB/s. Total sequential write bandwidth is 383.4 MiB/s > * At 12 pods, the average sequential write bandwidth per Longhorn volume is 31.8 MiB/s. Total sequential write bandwidth is 381.6 MiB/s > * At 15 pods, the average sequential write bandwidth per Longhorn volume is 25.4 MiB/s. Total sequential write bandwidth is 381 MiB/s drawing **Analysis & Conclusion**: * From the scaling test so far, we can see that the total sequential write bandwidth of all Longhorn volumes remain relatively the same around 380 MiB/s when the number of Kbench pods increase If we call the average sequential write bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 380. Users can use this information to make some prediction for this cluster: * The upper bound limit that Longhorn system can achieve in this cluster is 380 MiB/s sequential write bandwidth * If each workload pod is doing 7.45 MiB/s random read bandwidth on average, there can be 51 pods * When the user keeps scaling up number of pods, eventually this reciprocal relation (x * y = 380) might no longer hold as the CPU contention and other factors kick in (i.e. x*y will be less and less) * The bottleneck in this cluster seems to be the IOPs performance of the EBS volumes on host instead of CPU, memory, or network bandwidth ### Sequential Write Bandwidth - Rate Limited In this test, we use 1 control plane node + 3 worker nodes. We add a rate limit to Kbench so that each Kbench pod is only doing 7.45 MiB/s sequential write bandwidth and observe the performance of Longhorn volumes. Then we scale up the number of Kbench pods to 51 to see if the system is able to achieve 379.9 MiB/s sequential write bandwidth.
We use this yaml manifest for the Kbench workload:
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-bandwidth" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" - name: RATE value: ",7629k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
**Result**: > * The sequential write bandwidth per Longhorn volume is 7.48 MiB/s > * The total sequential write bandwidth can be achieved by all 51 Longhorn volumes is 381.5 MiB/s drawing **Analysis & Conclusion**: * As discussed in the [Sequential Write Bandwidth - Stress Tests](#sequential-write-bandwidth---stress-tests) section above, we come up with the conclusion that `If we call the sequential write bandwidth each volume can achieve (x) and the number of volumes (y), they form a reciprocal function: x * y = 380.` * The test result here further confirm this conclusion as x * y = 7.48*51 = 381.5 ### Random Write Latency - Stress Tests In this test, we use a cluster that has 1 control plane node + 3 worker nodes. First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: ",7559k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "random-write-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: ",7559k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: random-write-latency`: specifies that we are running [random write latency job](https://github.com/longhorn/kbench/blob/main/fio/latency-random-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 810 microsecond > * Longhorn: 2040 microsecond drawing **Analysis & Conclusion**: * Because the IO path of a Longhorn volume is longer than local path provisioner, it is expected that the Longhorn volume has bigger random write latency Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how random write latency is affected when there are more and more pods. **Result**: | Number of Kbench pods | Average Random Write Latency (ns) | |-----------------------|-----------------------------------| | 1 | 2039519 | | 3 | 2056904 | | 6 | 2091819 | | 9 | 2135070 | | 12 | 2253089 | | 15 | 2807773 | | 18 | 3373824 | | 21 | 3938172 | | 24 | 4503630 | | 27 | 5071816 | | 30 | 5640033 | | 33 | 6208153 | | 36 | 6771033 | | 39 | 7341914 | | 42 | 7905373 | | 45 | 8480925 | | 48 | 9040640 | | 51 | 9609189 | drawing **Analysis & Conclusion**: * As the number of kbench pods increase: * From 0 to 12 pods, the average random write latency of each Longhorn volume increase in linear fashion with smaller rate * From 12 to 51 pods, the average random write latency of each Longhorn volume increase in linear fashion with bigger rate ### Sequential Write Latency - Stress Tests In this test, we use a cluster that has 1 control plane node + 3 worker nodes. First, we do a comparison between a single RWO Longhorn PVC against a single Local Path Provisioner volume. We deploy 1 Kbench pod which attaches a Local Path Provisioner PVC. Then we delete the above Kbench pod and PVC and repeat the test with Longhorn PVC instead. We use this yaml manifest for the Kbench workload:
With local path provisioner storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: ",7559k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "local-path" resources: requests: storage: 20Gi ```
With longhorn storageclass
```yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: test-sts namespace: default spec: serviceName: test-sts replicas: 1 selector: matchLabels: app: test-sts podManagementPolicy: Parallel template: metadata: labels: app: test-sts spec: containers: - name: kbench image: phanle1010/kbench:dev imagePullPolicy: Always env: - name: MODE value: "sequential-write-latency" - name: OUTPUT value: /test-result/device - name: FILE_NAME value: "/volume/test" - name: SIZE value: "15G" - name: CPU_IDLE_PROF value: "disabled" - name: SKIP_PARSE value: "true" - name: LONG_RUN value: "true" # - name: RATE # value: ",7559k" volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result - name: metric-exporter image: phanle1010/kbench:dev imagePullPolicy: Always readinessProbe: exec: command: - sh - -c - '[ "$(ls -A /test-result)" ]' initialDelaySeconds: 22 periodSeconds: 10 command: - metric-exporter - -d - start env: - name: DATA_DIR value: /test-result - name: VOLUME_ACCESS_MODE value: rwo - name: TEST_MODE value: write-only - name: RATE_LIMIT_TYPE value: no-rate-limit ports: - containerPort: 8080 name: metrics volumeMounts: - name: vol mountPath: /volume/ - name: shared-data mountPath: /test-result volumes: - name: shared-data emptyDir: {} volumeClaimTemplates: - metadata: name: vol spec: accessModes: ["ReadWriteOnce"] storageClassName: "longhorn" resources: requests: storage: 20Gi ```
Some of the important Kbench parameters we would like to call out are: * `MODE: sequential-write-latency`: specifies that we are running [sequential write latency job](https://github.com/longhorn/kbench/blob/main/fio/latency-sequential-write.fio) * `SIZE: 15G`: the `fio` test size is 15G, this will avoid cache effect * PVC size is 20G **Result**: > * Local Path Provisioner: 803 microsecond > * Longhorn: 2028 microsecond drawing **Analysis & Conclusion**: * Because the IO path of a Longhorn volume is longer than local path provisioner, it is expected that the Longhorn volume has bigger sequential write latency Next, we use Kbench with Longhorn PVCs and scale up the number of Kbench pods to see how sequential write latency is affected when there are more and more pods. **Result**: | Number of Kbench pods | Average Sequential Write Latency (ns) | |-----------------------|---------------------------------------| | 1 | 2027403 | | 3 | 2064941 | | 6 | 2101946 | | 9 | 2135524 | | 12 | 2256725 | | 15 | 2812809 | | 18 | 3377877 | | 21 | 3942233 | | 24 | 4503992 | | 27 | 5078192 | | 30 | 5640632 | | 33 | 6204730 | | 36 | 6770942 | | 39 | 7343912 | | 42 | 7906260 | | 45 | 8475788 | | 48 | 9035704 | | 51 | 9611938 | drawing **Analysis & Conclusion**: * As the number of kbench pods increase: * From 0 to 12 pods, the average sequential write latency of each Longhorn volume increase in linear fashion with smaller rate * From 12 to 51 pods, the average sequential write latency of each Longhorn volume increase in linear fashion with bigger rate ## Longhorn Control Plane Performance We deploy a StatefulSet workload. Each pod of the StatefulSet is a nginx web server with 1 Longhorn PVC attached to it. Each pods has a liveness probe to ensure the Longhorn volume is still functional. This is the yaml of the StatefulSet:
StatefulSet workload
```yaml apiVersion: v1 kind: Service metadata: name: nginx labels: app: nginx spec: ports: - port: 80 name: web selector: app: nginx type: NodePort --- apiVersion: apps/v1 kind: StatefulSet metadata: name: web namespace: default spec: podManagementPolicy: Parallel selector: matchLabels: app: nginx serviceName: "nginx" replicas: 0 template: metadata: labels: app: nginx spec: restartPolicy: Always terminationGracePeriodSeconds: 10 containers: - name: nginx image: registry.k8s.io/nginx-slim:0.8 livenessProbe: exec: command: - ls - /usr/share/nginx/html/lost+found initialDelaySeconds: 5 periodSeconds: 5 ports: - containerPort: 80 name: web volumeMounts: - name: www mountPath: /usr/share/nginx/html volumeClaimTemplates: - metadata: name: www spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "longhorn" resources: requests: storage: 1Gi ```
Then we scale up the StatefulSet with the rate 30 pods per minute and monitor the number of pods, CPU and RAM usage of Longhorn system, ECTD, and KubeAPI metrics. The scaling script is: ```bash NAMESPACE="default"; STATEFULSET_NAME="web"; MAX_PODS=""; INTERVAL=2; while true; do CURRENT_REPLICAS=$(kubectl get statefulset -n $NAMESPACE $STATEFULSET_NAME -o jsonpath='{.spec.replicas}'); if [ "$CURRENT_REPLICAS" -lt "$MAX_PODS" ]; then NEW_REPLICAS=$((CURRENT_REPLICAS + 1)); echo "Scaling $STATEFULSET_NAME to $NEW_REPLICAS replicas"; kubectl scale --replicas=$NEW_REPLICAS statefulset -n $NAMESPACE $STATEFULSET_NAME; fi; sleep $INTERVAL; done ``` We define the following stopping conditions for the scale test. If we hit any of these conditions, we stop the scale test and record the current scaling number. 1. There are more than 10 pods with starting time more than 4 minutes (the duration from pod being created to pod becomes running). The idea is that if pod is taking more than 4 minutes to start, the system is being too slow 1. The per-minute rate of pod become running is smaller than 10. It means that the system should be able to create more than 10 pods per minute. If not, it is being too slow 1. There are more than 3 crashing pods or restarted containers. If there are many crashes, the system is not stable at the scale Finally, we will scale down the StatefulSet from the max value to 0 with the rate of 30 pods per minute. We will measure the scaling down speed. This test design touches the end-to-end life cycle of a pod from pod creation -> CSI flow provision/attach/mount Longhorn volume -> CSI flow unmount/detach Longhorn volume -> pod termination > Notes: There are a few small modification we made in this section comparing to the previous data plan performance testing section: > 1. Because of this known issue in Longhorn v1.6.0 https://github.com/longhorn/longhorn/issues/7919, it might lock up the Longhorn control plane when there are many volumes. > Therefore, we decided to use Longhorn v1.6.1 version for this control plane test > Longhorn v1.6.1 should have very similar characteristic as Longhorn 1.6.0 except for some bug fixes > 1. At this large scale, Prometheus server uses too much RAM and CPU. > Therefore, we decided to create a dedicated node for the Prometheus server to avoid interfering with Longhorn system > 1. We set the Longhorn setting [automatically-delete-workload-pod-when-the-volume-is-detached-unexpectedly](https://longhorn.io/docs/1.6.1/references/settings/#automatically-delete-workload-pod-when-the-volume-is-detached-unexpectedly) to false so that we can observe the workload pod crashes instead of asking Longhorn to clean it up quickly ### 1 master node + 3 worker nodes #### Result 1. The cluster can handle up to 699 pods with 699 Longhorn volumes 1. Each node can have 233 Longhorn volumes. 2. Each node can have 669 Longhorn replicas. #### Scaling Up drawing ##### Comment 1. This test hits the 1st stopping condition (there are more than 10 pods with starting time more than 4 minutes) at 699 pods (with 699 Longhorn volumes). 1. We can see that the number of running pod is linearly scaled from 0 to 460. Then rate is slow down after that. 1. The component which is consuming the most CPU and RAM are instance manager pods followed by longhorn manager pods 1. API and ETCD are not overloaded #### Scaling Down drawing ##### Comment Scaling down has no issue. Everything is perfectly linear > Note: that some of the graphs are not very meaningful because they reflect starting time and no pods are starting. ### 1 master node + 6 worker nodes #### Result 1. The cluster can handle up to 1114 pods with 1114 Longhorn volumes 1. Each node can have 185 Longhorn volumes 2. Each node can have 557 Longhorn replicas #### Scaling Up drawing ##### Comment 1. This test hits the 1st stopping condition (there are more than 10 pods with starting time more than 4 minutes) at 1114 pods (with 1114 Longhorn volumes) 1. We can see that the number of running pod is linearly scaled from 0 to 840. Then rate is slow down after that 1. The component which is consuming the most CPU and RAM are instance manager pods followed by longhorn manager pods 1. API and ETCD are not overloaded #### Scaling Down ##### Comment Scaling down has no issue. Everything is perfectly linear similar to the above test ### 1 master node + 9 worker nodes #### Result 1. The cluster can handle up to 1561 pods with 1561 Longhorn volumes 1. Each node can have 173 Longhorn volumes 2. Each node can have 520 Longhorn replicas #### Scaling Up drawing ##### Comment 1. This test hits the 1st stopping condition (there are more than 10 pods with starting time more than 4 minutes) at 1561 pods (with 1561 Longhorn volumes) 1. We can see that the number of running pod is linearly scaled from 0 to 1400. Then rate is slow down after that 1. The component which is consuming the most CPU and RAM are instance manager pods followed by longhorn manager pods 1. API and ETCD are not overloaded #### Scaling Down ##### Comment Scaling down has no issue. Everything is perfectly linear similar to the above test ## Volume Maximum Size As mentioned above the maximum Longhorn volume size is limited by the replica rebuilding time. Currently, Longhorn limited the rebuilding time to be 24h. Therefore, we will measure how much data we can rebuild within 24h. We further notice that the rebuilding speed is depended on the data pattern inside the volume and the number of snapshot inside the volume. ### Case 1: Volume With Full Data 1. We deploy a volume of 100G size with 1 replica 1. Write 100G of data to the volume by `dd if=/dev/urandom of=/dev/longhorn/ bs=1M count=102400` 1. Scale up the number of replicas to 2 to trigger rebuilding 1. Measure the time it takes to finish rebuilding. We use this python script to draw the rebuilding speed graph:
rebuilding_speed_monitor.py
```python #!/usr/bin/env python3 from kubernetes import client, config from datetime import datetime import time import matplotlib.pyplot as plt def main(): # Load kubeconfig config.load_kube_config() namespace = "longhorn-system" engine_name = "testvol-e-0" # replace by the name of the target volume's engine # Create a custom object API client api_instance = client.CustomObjectsApi() starting_timestamp = datetime.now() data = [] rebuild_started = False while True: current_duration = (datetime.now() - starting_timestamp).total_seconds() current_progress = 0 try: # Get the engine object engine = api_instance.get_namespaced_custom_object( group="longhorn.io", version="v1beta1", namespace=namespace, plural="engines", name=engine_name ) # Extract the rebuildStatus field rebuild_status = engine.get("status", {}).get("rebuildStatus", {}) # Extract the rebuild progress for _, status in rebuild_status.items(): for k, v in status.items(): if k == "progress": current_progress = int(v) data.append((current_duration, current_progress)) if current_progress > 0: rebuild_started = True if rebuild_started and (current_progress <= 0 or current_progress >= 100): print("Done. Sleeping for 1 hour to give chance for user to take screenshot") time.sleep(3600) return print("%f : %d" % (current_duration, current_progress)) plot_graph(data) except client.exceptions.ApiException as e: print(f"Error getting engine: {e}") time.sleep(2) # Function to plot the graph def plot_graph(data_points): # Extract x and y coordinates x = [point[0] for point in data_points] y = [point[1] for point in data_points] # Clear the previous plot plt.clf() # Plot the graph plt.plot(x, y) # Add labels and title plt.xlabel('Time (seconds)') plt.ylabel('Progress (%)') plt.title('Rebuilding Speed') # Draw the plot without blocking plt.pause(0.001) # You can adjust this value to control the refresh rate if __name__ == "__main__": main() ```
drawing 1. So it takes 281 seconds to rebuild the volume. The maximum size of Longhorn volume would be calculated as `(3600*24*100)/281 = 30747 Gi` (30Ti) > Note: If the volume has more than 1 snapshot, the maximum size would be smaller. > More specifically, let's say the volume is set so that it has maximum x snapshots (including volume-head). > The maximum volume size would be calculated as `30Ti/x` ### Case 2: Volume With Data Pattern 4k_data-4k_hole-4k_data-4k_hole-... From our experience, this data pattern is the worst case for rebuilding speed. 1. We deploy a volume of 10G size with 1 replica 1. Use this `fio` job to generate data pattern 4k_data-4k_hole-4k_data-4k_hole-...
job.fio
```bash [global] bs=4k iodepth=128 direct=1 end_fsync=1 ioengine=libaio randseed=1 randrepeat=0 group_reporting # 4k bs [job1] bs=4k rw=write:4k filename=/dev/longhorn/ # replace by volume name name=data4k-holes4k io_size=5G ```
1. Scale up the number of replicas to 2 to trigger rebuilding 1. Measure the time it takes to finish rebuilding. We use the same python script above to draw the rebuilding speed graph: drawing 1. So it takes 1118 seconds to rebuild the volume. From the graph we see that the rebuilding speed is perfectly linear. Therefore, we can predict that the maximum size of Longhorn volume would be calculated as `(3600*24*10)/1118 = 772 Gi` > Note: If the volume has snapshots, the maximum size would be smaller. > More specifically, let's say the volume is set so that it has maximum x snapshots (including volume-head). > The maximum volume size would be calculated as `772Gi/x` ## FAQs The following are some FAQs that we can answer using data from this report. Note that the answers are particular to this setup. If you are more/less powerful setup, the number might be different. 1. What is the maximum number of Longhorn volumes inside the cluster? 1. For 1 master node + 3 worker nodes cluster: 699 Longhorn volumes 1. For 1 master node + 6 worker nodes cluster: 1114 Longhorn volumes 1. For 1 master node + 9 worker nodes cluster: 1561 Longhorn volumes 1. We don't test with bigger number of nodes, but you can have roughly estimation using the above data 1. What is the maximum number of Longhorn volumes can be attached to a node? Answer: 233 Longhorn volumes 1. What is the maximum number of Longhorn replicas per node? Answer: 669 Longhorn replicas 1. What is the maximum Longhorn volume size? 1. For volume with full data pattern (A.K.A every snapshot is a full), the maximum volume size would be calculated as `30Ti/x` where `x` is maximum number of snapshots setting of the volume (including volume-head) 1. For volume with data pattern 4k_data-4k_hole-4k_data-4k_hole-..., the maximum volume size would be calculated as `772Gi/x` where `x` is maximum number of snapshots setting of the volume (including volume-head). This is the worst case 1. What is the maximum IOPs of a Longhorn volume? 1. random-read-iops: 22180 1. sequential-read-iops: 39772 1. random-write-iops: 1150 1. sequential-write-iops: 2340 1. What is the maximum bandwidth of a Longhorn volume? 1. random-read-bandwidth: 874 MiB/s 1. sequential-read-bandwidth: 669 MiB/s 1. random-write-bandwidth: 159 MiB/s 1. sequential-write-bandwidth: 291 MiB/s 1. What is the minimum latency of Longhorn volume? 1. random-read-latency: 750 microseconds 1. sequential-read-latency: 740 microseconds 1. random-write-latency: 2040 microseconds 1. sequential-write-latency: 2028 microseconds 1. What is the maximum IOPs of all Longhorn volumes with 3 worker nodes? 1. random-read-iops: ~31700 1. sequential-read-iops: ~62500 1. random-write-iops: ~5250 1. sequential-write-iops: ~10400 1. Also, the number is linearly increased by number of worker nodes 1. What is the maximum bandwidth of all Longhorn volumes with 3 worker nodes? 1. random-read-bandwidth: ~1170 MiB/s 1. sequential-read-bandwidth: ~1152 MiB/s 1. random-write-bandwidth: ~376.8 MiB/s 1. sequential-write-bandwidth: ~381.6 MiB/s 1. Also, the number is linearly increased by number of worker nodes ================================================ FILE: scripts/generate-backupstore-credentials.sh ================================================ #!/bin/bash set -euo pipefail ######################################### # Credential Configuration ######################################### # Azurite / AzBlob : "${AZBLOB_ACCOUNT_NAME:=}" : "${AZBLOB_ACCOUNT_KEY:=}" : "${AZBLOB_ENDPOINT:=}" # CIFS : "${CIFS_USERNAME:=}" : "${CIFS_PASSWORD:=}" # MinIO / S3-compatible : "${AWS_ACCESS_KEY_ID:=}" : "${AWS_SECRET_ACCESS_KEY:=}" : "${AWS_ENDPOINTS:=}" : "${AWS_CERT:=}" : "${AWS_CERT_KEY:=}" ######################################### readonly SUPPORTED_BACKENDS=("azurite" "cifs" "minio" "nfs") # Always work relative to the repo root SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ALL_TARGET_DIR="${PROJECT_ROOT}/deploy/backupstores/overlays/generated-credentials" check_env_or_fail() { local var_name="$1" if [[ -z "${!var_name:-}" ]]; then echo "ERROR: Environment variable '$var_name' is not set or empty." >&2 exit 1 fi } generate_all_overlay() { local ALL_DIR="${ALL_TARGET_DIR}/all" rm -rf "${ALL_DIR:?}" && mkdir -p "${ALL_DIR}" { echo "apiVersion: kustomize.config.k8s.io/v1beta1" echo "kind: Kustomization" echo "" echo "resources:" for backend in "${SUPPORTED_BACKENDS[@]}"; do echo " - ../${backend}" done } > "${ALL_DIR}/kustomization.yaml" echo "Unified overlay generated at: ${ALL_DIR}" } generate_backend() { local backend=$1 TARGET_DIR="${ALL_TARGET_DIR}/${backend}" rm -rf "${TARGET_DIR:?}" && mkdir -p "${TARGET_DIR}" case "$backend" in azurite) generate_azurite_backend "$TARGET_DIR" ;; cifs) generate_cifs_backend "$TARGET_DIR" ;; minio) generate_minio_backend "$TARGET_DIR" ;; nfs) generate_nfs_backend "$TARGET_DIR" ;; *) echo "Unsupported backend: $backend" exit 1 ;; esac echo "Credentials for $backend generated at: ${TARGET_DIR}" } generate_azurite_backend() { local TARGET_DIR=$1 check_env_or_fail AZBLOB_ACCOUNT_NAME check_env_or_fail AZBLOB_ACCOUNT_KEY check_env_or_fail AZBLOB_ENDPOINT generate_patch_with_ns longhorn-system azblob-secret azurite-backupstore-secret \ AZBLOB_ACCOUNT_NAME "$AZBLOB_ACCOUNT_NAME" \ AZBLOB_ACCOUNT_KEY "$AZBLOB_ACCOUNT_KEY" \ AZBLOB_ENDPOINT "$AZBLOB_ENDPOINT" cat < "${TARGET_DIR}/kustomization.yaml" apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../base/azurite patches: - path: azurite-backupstore-secret-patch-longhorn-system.yaml EOF } generate_cifs_backend() { local TARGET_DIR=$1 check_env_or_fail CIFS_USERNAME check_env_or_fail CIFS_PASSWORD generate_patch_with_ns longhorn-system cifs-secret cifs-backupstore-secret \ CIFS_USERNAME "$CIFS_USERNAME" \ CIFS_PASSWORD "$CIFS_PASSWORD" generate_patch_with_ns default cifs-secret cifs-backupstore-secret \ CIFS_USERNAME "$CIFS_USERNAME" \ CIFS_PASSWORD "$CIFS_PASSWORD" cat < "${TARGET_DIR}/kustomization.yaml" apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../base/cifs patches: - path: cifs-backupstore-secret-patch-longhorn-system.yaml - path: cifs-backupstore-secret-patch-default.yaml EOF } generate_minio_backend() { local TARGET_DIR=$1 check_env_or_fail AWS_ACCESS_KEY_ID check_env_or_fail AWS_SECRET_ACCESS_KEY check_env_or_fail AWS_ENDPOINTS if $BASE64_ENCODE; then if [[ "$AWS_ENDPOINTS" == https://* ]]; then check_env_or_fail AWS_CERT check_env_or_fail AWS_CERT_KEY fi else if ! decoded_endpoint=$(echo "$AWS_ENDPOINTS" | base64 --decode 2>/dev/null); then echo "ERROR: Failed to decode AWS_ENDPOINTS. Must be valid base64." >&2 exit 1 fi if [[ "$decoded_endpoint" == https://* ]]; then check_env_or_fail AWS_CERT check_env_or_fail AWS_CERT_KEY fi fi generate_patch_with_ns longhorn-system minio-secret minio-backupstore-secret \ AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" \ AWS_ENDPOINTS "$AWS_ENDPOINTS" \ AWS_CERT "$AWS_CERT" \ AWS_CERT_KEY "$AWS_CERT_KEY" generate_patch_with_ns default minio-secret minio-backupstore-secret \ AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" \ AWS_ENDPOINTS "$AWS_ENDPOINTS" \ AWS_CERT "$AWS_CERT" \ AWS_CERT_KEY "$AWS_CERT_KEY" cat < "${TARGET_DIR}/kustomization.yaml" apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../base/minio patches: - path: minio-backupstore-secret-patch-longhorn-system.yaml - path: minio-backupstore-secret-patch-default.yaml EOF } generate_nfs_backend() { local TARGET_DIR=$1 cat < "${TARGET_DIR}/kustomization.yaml" apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../base/nfs EOF } generate_patch_with_ns() { local ns=$1 local name=$2 local file=$3 shift 3 { echo "apiVersion: v1" echo "kind: Secret" echo "metadata:" echo " name: $name" echo " namespace: $ns" echo "type: Opaque" echo "data:" while [[ $# -gt 1 ]]; do key=$1 val=$2 fail_if_base64_encoded "$key" "$val" echo " $key: $(b64 "$val")" shift 2 done } > "${TARGET_DIR}/${file}-patch-${ns}.yaml" } b64() { if $BASE64_ENCODE; then echo -n "$1" | base64 | tr -d '\n' else echo -n "$1" fi } fail_if_base64_encoded() { local key="$1" local val="$2" if $BASE64_ENCODE && is_base64 "$val"; then echo "ERROR: Input for $key appears to be already base64-encoded. Refusing to double-encode." >&2 echo "Hint: Use --no-encode if your input is already base64." >&2 exit 1 fi } is_base64() { echo "$1" | base64 --decode >/dev/null 2>&1 } # Entry point BACKEND="" BASE64_ENCODE=true while [[ $# -gt 0 ]]; do case "$1" in --no-encode) BASE64_ENCODE=false ;; azurite|cifs|minio|nfs|all) BACKEND=$1 ;; *) echo "Unknown option or argument: $1" echo "Usage: $0 [azurite|cifs|minio|nfs|all] [--no-encode]" exit 1 ;; esac shift done if [[ -z "$BACKEND" ]]; then echo "Error: Must specify one of: azurite, cifs, minio, nfs or all" echo "Usage: $0 [azurite|cifs|minio|nfs|all] [--no-encode]" exit 1 fi if $BASE64_ENCODE; then echo "Base64 encoding: enabled" else echo "Base64 encoding: disabled" fi if [[ "$BACKEND" == "all" ]]; then for backend in "${SUPPORTED_BACKENDS[@]}"; do generate_backend "$backend" done generate_all_overlay else generate_backend "$BACKEND" fi ================================================ FILE: scripts/generate-longhorn-yaml.sh ================================================ #!/usr/bin/env bash set -o errexit set -o xtrace PRJ_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null || realpath "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null) CHART_DIR="$PRJ_DIR/chart" DEPLOY_YAMLS=("$PRJ_DIR/deploy/longhorn.yaml" "$PRJ_DIR/deploy/longhorn-okd.yaml") DEPLOY_YAML_TMP="$PRJ_DIR/deploy/longhorn.yaml.tmp" NAMESPACE=${NAMESPACE:-longhorn-system} if ! command -v helm &> /dev/null || ! helm version --short | grep -q "v3\|v4"; then echo "Please install helm v4 first before generating $DEPLOY_YAML!" exit 1 fi for DEPLOY_YAML in ${DEPLOY_YAMLS[@]}; do cat < "$DEPLOY_YAML" --- # Builtin: "helm template" does not respect --create-namespace apiVersion: v1 kind: Namespace metadata: name: $NAMESPACE EOD if [[ $DEPLOY_YAML == $PRJ_DIR/deploy/longhorn-okd.yaml ]]; then OKD_ENABLED_FLAG="--set openshift.enabled=true" fi helm template longhorn "$CHART_DIR" --namespace "$NAMESPACE" $OKD_ENABLED_FLAG --create-namespace --no-hooks >>"$DEPLOY_YAML" < "$DEPLOY_YAML" grep -v 'helm.sh\|app.kubernetes.io/managed-by: Helm' | grep -v "helm.sh/chart:" > "$DEPLOY_YAML_TMP" mv "$DEPLOY_YAML_TMP" "$DEPLOY_YAML" done ================================================ FILE: scripts/helm-docs.sh ================================================ #!/usr/bin/env bash ## Reference: https://github.com/norwoodj/helm-docs set -o errexit set -o xtrace PRJ_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null || realpath "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null) CHART_DIR="$PRJ_DIR/chart" echo "$CHART_DIR" echo "Running Helm-Docs" sudo docker run \ -v "$CHART_DIR:/helm-docs" \ -u $(id -u) \ jnorwood/helm-docs:v1.9.1 ================================================ FILE: scripts/lhexec ================================================ #!/usr/bin/env bash NS="longhorn-system" print_usage() { echo "Usage: ${0} [|-h|--help] volume_name longhorn_commands_arguments" echo "" echo "Examples:" echo " ${0} test-vol snapshot ls" echo " ${0} test-vol info" echo "" echo "Note: Must have Longhorn installed in "longhorn-system" namespace and have access to "kubectl" and the namespace" echo "" exit 0 } check_volume_exist(){ VOLUME_NAME=${1} kubectl -n ${NS} get lhv ${VOLUME_NAME} > /dev/null 2>&1 if [[ ${?} -ne 0 ]]; then echo "Err: Volume ${VOLUME_NAME} not found" exit 1 fi } check_engine_state(){ VOLUME_NAME=${1} LHE_STATE_FILTER="{.items[?(@.spec.volumeName==\"${VOLUME_NAME}\")].status.currentState}" LHE_STATE=`kubectl -n ${NS} get lhe --output=jsonpath="${LHE_STATE_FILTER}"` if [[ ${LHE_STATE} != "running" ]]; then echo "Err: Longhorn engine for volume ${VOLUME_NAME} is not running" exit 1 fi } exec_command() { VOLUME_NAME=${1} COMMAND_ARGS="${@:2}" INSTANCE_MANAGER_NAME_FILTER="{.items[?(@.spec.volumeName==\"${VOLUME_NAME}\")].status.instanceManagerName}" INSTANCE_MANAGER_NAME=`kubectl -n ${NS} get lhe --output=jsonpath="${INSTANCE_MANAGER_NAME_FILTER}"` ENGINE_PORT_FILTER="{.items[?(@.spec.volumeName==\"${VOLUME_NAME}\")].status.port}" ENGINE_PORT=`kubectl -n ${NS} get lhe --output=jsonpath="${ENGINE_PORT_FILTER}"` LONGHORN_BIN_PATH=`kubectl -n ${NS} exec -it ${INSTANCE_MANAGER_NAME} -- bash -c "ps -eo command | grep \" ${VOLUME_NAME} \" | grep -v grep | awk '{ printf(\"%s\", \\$1)}'"` kubectl -n ${NS} exec -it ${INSTANCE_MANAGER_NAME} -- bash -c "${LONGHORN_BIN_PATH} --url localhost:${ENGINE_PORT} ${COMMAND_ARGS}" } ARG=$1 case $ARG in "" | "-h" | "--help") print_usage ;; *) VOLUME_NAME=${ARG} shift COMMAND_ARGS="${@}" if [[ ${COMMAND_ARGS} == "" ]]; then COMMAND_ARGS="help" fi check_volume_exist ${VOLUME_NAME} check_engine_state ${VOLUME_NAME} exec_command ${VOLUME_NAME} ${COMMAND_ARGS} ;; esac ================================================ FILE: scripts/load-images.sh ================================================ #!/usr/bin/env bash list="longhorn-images.txt" CONTAINER_CLI=${CONTAINER_CLI:-docker} POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in -r|--registry) reg="$2" shift # past argument shift # past value ;; -l|--image-list) list="$2" shift # past argument shift # past value ;; -i|--images) images="$2" shift # past argument shift # past value ;; -h|--help) help="true" shift ;; *) echo "Error! invalid flag: ${key}" help="true" break ;; esac done usage () { echo "USAGE: $0 [--image-list longhorn-images.txt] [--images longhorn-images.tar.gz] --registry my.registry.com:5000" echo " [-l|--images-list path] text file with list of images. 1 per line." echo " [-i|--images path] tar.gz generated by docker/podman save. If empty, the script will try to find images in local images" echo " [-r|--registry registry:port] target private registry:port. By default, registry is Docker Hub" echo " [-h|--help] Usage message" echo "" echo "To use podman instead of docker set the environment variable CONTAINER_CLI=podman" } if [[ $help ]]; then usage exit 0 fi if [[ -n $reg ]]; then reg+="/" fi set -e -x if [[ $images ]]; then $CONTAINER_CLI load --input ${images} fi for i in $(cat ${list}); do case $i in */*/*) $CONTAINER_CLI tag ${i} ${reg}longhornio/${i#*/*/} $CONTAINER_CLI push ${reg}longhornio/${i#*/*/} ;; */*) $CONTAINER_CLI tag ${i} ${reg}longhornio/${i#*/} $CONTAINER_CLI push ${reg}longhornio/${i#*/} ;; *) $CONTAINER_CLI tag ${i} ${reg}longhornio/${i} $CONTAINER_CLI push ${reg}longhornio/${i} ;; esac done ================================================ FILE: scripts/longhorn_rancher_chart_migration.sh ================================================ #!/usr/bin/env bash #set -x kubectl get-all version &> /dev/null if [ $? -ne 0 ]; then echo "ERROR: command (kubectl get-all) is not found. Please install it here: https://github.com/corneliusweig/ketall#installation" exit 1 fi set -e usage() { echo "" echo "The migration includes:" echo "1. Running the script with --type migrate to migrate the labels and annotations for Longhorn resources" echo "2. Manually installing Longhorn chart in app&marketplace UI" echo "3. Running script with --type cleanup to remove the old Longhorn chart from old catalog UI" echo "" echo "usage:" echo "$0 [options]" echo " -u | --upstream-kubeconfig: upstream rancher cluster kubeconfig path" echo " -d | --downstream-kubeconfig: downstream cluster kubeconfig path" echo " -t | --type: specify the type you want to run (migrate or cleanup)" echo " --dry-run: do not run migriation" echo "" echo "example:" echo " $0 -u /path/to/upstream/rancher/cluster/kubeconfig -d /path/to/downstream/cluster/kubeconfig" } SCRIPT_DIR="$(dirname "$0")" UPSTREAM_KUBECONFIG="" DOWNSTREAM_KUBECONFIG="" KUBECTL_DRY_RUN="" while [ "$1" != "" ]; do case $1 in -u | --upstream-kubeconfig) shift UPSTREAM_KUBECONFIG="$1" ;; -d | --downstream-kubeconfig) shift DOWNSTREAM_KUBECONFIG="$1" ;; -t | --type) shift TYPE="$1" ;; --dry-run) KUBECTL_DRY_RUN="--dry-run=client" ;; *) usage exit 1 ;; esac shift done if [ -z "$UPSTREAM_KUBECONFIG" ]; then echo "--upstream-kubeconfig is mandatory" usage exit 1 fi if [ -z "$DOWNSTREAM_KUBECONFIG" ]; then echo "--downstream-kubeconfig is mandatory" usage exit 1 fi if [ "$TYPE" != "migrate" ] && [ "$TYPE" != "cleanup" ] ; then echo "--type must be set to migrate or cleanup" usage exit 1 fi # Longhorn Namespace RELEASE_NAMESPACE=longhorn-system # Longhorn Release Name RELEASE_NAME=longhorn-system echo "Looking up Rancher Project App '${RELEASE_NAME}' ..." DOWNSTREAMCLUSTERID=$(cat ${DOWNSTREAM_KUBECONFIG} | grep "server:.*https://.*/k8s/clusters/.*" | awk -F'/' '{print $(NF)}' | awk -F'"' '{print $1}') RANCHERAPP=$(kubectl --kubeconfig ${UPSTREAM_KUBECONFIG} get --all-namespaces apps.project.cattle.io -o jsonpath='{range.items[*]}{.metadata.namespace} {.metadata.name} {.spec.targetNamespace} {.spec.projectName} {.spec.externalId}{"\n"}{end}' | grep -s "${RELEASE_NAME} ${RELEASE_NAMESPACE} ${DOWNSTREAMCLUSTERID}") RANCHERAPPNS=$(echo "${RANCHERAPP}" | awk '{print $1}') RANCHERAPPEXTERNALID=$(echo "${RANCHERAPP}" | awk '{print $5}') RANCHERAPPCATALOG=$(echo "${RANCHERAPPEXTERNALID}" | sed -n 's/.*catalog=\(.*\)/\1/p' | awk -F '&' '{print $1}' | sed 's/migrated-//') RANCHERAPPTEMPLATE=$(echo "${RANCHERAPPEXTERNALID}" | sed -n 's/.*template=\(.*\)/\1/p' | awk -F '&' '{print $1}') RANCHERAPPTEMPLATEVERSION=$(echo "${RANCHERAPPEXTERNALID}" | sed -n 's/.*version=\(.*\)/\1/p' | awk -F '&' '{print $1}') RANCHERAPPVALUES="" RANCHERAPPANSWERS="" if [ -z "$DOWNSTREAMCLUSTERID" ] || [ -z "$RANCHERAPP" ] || [ -z "$RANCHERAPPNS" ] || [ -z "$RANCHERAPPCATALOG" ] || [ -z "$RANCHERAPPTEMPLATE" ] || [ -z "$RANCHERAPPTEMPLATEVERSION" ]; then echo "Rancher Project App '${RELEASE_NAME}' not found!" exit 1 fi RANCHERAPPVALUES=$(kubectl --kubeconfig ${UPSTREAM_KUBECONFIG} -n ${RANCHERAPPNS} get apps.project.cattle.io ${RELEASE_NAME} -o go-template='{{if .spec.valuesYaml}}{{.spec.valuesYaml}}{{end}}') if [ -z "${RANCHERAPPVALUES}" ]; then RANCHERAPPANSWERS=$(kubectl --kubeconfig ${UPSTREAM_KUBECONFIG} -n ${RANCHERAPPNS} get apps.project.cattle.io ${RELEASE_NAME} -o go-template='{{if .spec.answers}}{{range $key,$value := .spec.answers}}{{$key}}: {{$value}}{{"\n"}}{{end}}{{end}}' | sed 's/: /=/' | sed 's/$/,/' | sed '$ s/.$//' | tr -d '\n') fi if [ -z "${RANCHERAPPVALUES:-$RANCHERAPPANSWERS}" ]; then echo "No valid answers found!" exit 1 fi echo "" echo "Rancher Project App '${RELEASE_NAME}' found:" echo " Project-Namespace: ${RANCHERAPPNS}" echo " Downstream-Cluster: ${DOWNSTREAMCLUSTERID}" echo " Catalog: ${RANCHERAPPCATALOG}" echo " Template: ${RANCHERAPPTEMPLATE} (${RANCHERAPPTEMPLATEVERSION})" echo " Answers:" printf '%s\n' "${RANCHERAPPVALUES:-$RANCHERAPPANSWERS}" echo "" if [ "$TYPE" == "cleanup" ] ; then MANAGER=$(kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} get ds longhorn-manager -ojsonpath="{.metadata.labels['app\.kubernetes\.io/managed-by']}") if [ $MANAGER != "Helm" ] ; then echo "Labels have not been migrated. Did you run the part 1 by specifying the flag --type migrate ?" exit 1 fi echo "" echo "Patching Project App Catalog ..." kubectl --kubeconfig ${UPSTREAM_KUBECONFIG} -n ${RANCHERAPPNS} ${KUBECTL_DRY_RUN} patch apps.project.cattle.io ${RELEASE_NAME} --type=merge --patch-file=/dev/stdin <<-EOF { "metadata": { "annotations": { "cattle.io/skipUninstall": "true", "catalog.cattle.io/ui-source-repo": "helm3-library", "catalog.cattle.io/ui-source-repo-type": "cluster", "apps.cattle.io/migrated": "true" } } } EOF if [ $? -ne 0 ]; then echo "Failed Patching Project App Catalog" exit 1 fi echo "" echo "Deleting Project App Catalog ..." kubectl --kubeconfig ${UPSTREAM_KUBECONFIG} -n ${RANCHERAPPNS} ${KUBECTL_DRY_RUN} delete apps.project.cattle.io ${RELEASE_NAME} exit 0 fi echo "" echo "" echo "Checking concurrent-automatic-engine-upgrade-per-node-limit setting ..." SETTING=$(kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} get settings.longhorn.io concurrent-automatic-engine-upgrade-per-node-limit -ojsonpath="{.value}") if [ "$SETTING" != "0" ]; then echo "concurrent-automatic-engine-upgrade-per-node-limit must be set to 0 before the migration" exit 1 fi echo "" echo "" echo "Looking up existing Resources ..." RESOURCES=$(kubectl get-all --kubeconfig ${DOWNSTREAM_KUBECONFIG} --exclude AppRevision -o name -l io.cattle.field/appId=${RELEASE_NAME} 2>/dev/null | sort) if [[ "$RESOURCES" == "No resources"* ]]; then RESOURCES="" fi echo "" echo "Patching CRD Resources ..." for resource in $RESOURCES; do if [[ $resource == "customresourcedefinition.apiextensions.k8s.io/"* ]]; then kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} ${KUBECTL_DRY_RUN} annotate --overwrite ${resource} "meta.helm.sh/release-name"="longhorn-crd" "meta.helm.sh/release-namespace"="${RELEASE_NAMESPACE}" "helm.sh/resource-policy"="keep" kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} ${KUBECTL_DRY_RUN} label --overwrite ${resource} "app.kubernetes.io/managed-by"="Helm" fi done echo "" echo "Patching Other Resources ..." for resource in $RESOURCES; do if [[ $resource == "customresourcedefinition.apiextensions.k8s.io/"* ]]; then continue fi kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} ${KUBECTL_DRY_RUN} annotate --overwrite ${resource} "meta.helm.sh/release-name"="longhorn" "meta.helm.sh/release-namespace"="${RELEASE_NAMESPACE}" kubectl --kubeconfig ${DOWNSTREAM_KUBECONFIG} -n ${RELEASE_NAMESPACE} ${KUBECTL_DRY_RUN} label --overwrite ${resource} "app.kubernetes.io/managed-by"="Helm" done echo "" echo "-----------------------------" echo "Successfully updated the annotations and labels for the resources!" echo "Next step:" echo " 1. Go to Rancher UI -> Go to the downstream cluster -> App&Marketplace -> Charts" echo " 2. Find and select the Longhorn chart" echo " 3. Select the chart version corresponding the Longhorn version ${RANCHERAPPTEMPLATEVERSION}" echo " 4. Install the chart with the correct helm values. Here are the helm values of your old charts: " printf '%s\n' "${RANCHERAPPVALUES:-$RANCHERAPPANSWERS}" echo " 5. Verify that the migrated charts are working ok" echo " 6. Run this script again with the flag --type cleanup to remove the old chart from the legacy UI" ================================================ FILE: scripts/migrate-for-pre-070-volumes.sh ================================================ #!/usr/bin/env bash NS="longhorn-system" print_usage() { echo "Usage: ${0} [ |-h|--help] [volume_name|--all]" echo "" echo "Examples:" echo " ${0} test-vol" echo " ${0} --all" echo "" echo "Note: Must have Longhorn installed in "longhorn-system" namespace" echo "" exit 0 } exec_command() { COMMAND_ARG="${@}" LONGHORN_MANAGER=$(kubectl -n ${NS} get po -l "app=longhorn-manager" | tr '\000' '\n' | sed -n '2p' | awk '{print $1}') kubectl -n "${NS}" exec -it "${LONGHORN_MANAGER}" -- longhorn-manager migrate-for-pre-070-volumes "${COMMAND_ARG}" } ARG=$1 case $ARG in "" | "-h" | "--help") print_usage ;; *) if [[ $# -ne 1 ]]; then echo "Command args number shouldn't be greater than 1" fi exec_command "${@}" ;; esac ================================================ FILE: scripts/restore-backup-to-file.sh ================================================ #!/usr/bin/env bash export RED='\x1b[0;31m' export NO_COLOR='\x1b[0m' usage () { echo "USAGE: $0 --aws-access-key \ " echo " --aws-secret-access-key \ " echo " --backup-url s3://backupbucket@ap-northeast-1/backupstore?backup=&volume= \ " echo " --output-file volume.raw \ " echo " --output-format raw \ " echo " --version " echo " --backing-file " echo "Restore a Longhorn backup to a raw image or a qcow2 image." echo "" echo " -u, --backup-url (Required) Backups S3/NFS URL. e.g., s3://backupbucket@us-east-1/backupstore?backup=backup-bd326da2c4414b02&volume=volumeexamplename" echo " -o, --output-file (Required) Output file, e.g., /tmp/restore/volume.raw" echo " -f, --output-format (Required) Output file format, e.g., raw or qcow2" echo " -v, --version (Required) Longhorn version, e.g., v1.3.2" echo " --aws-access-key (Optional) AWS credentials access key" echo " --aws-secret-access-key (Optional) AWS credentials access secret key" echo " --cifs-username (Optional) CIFS credentials username" echo " --cifs-password (Optional) CIFS credentials password" echo " -b, --backing-file (Optional) backing image. e.g., /tmp/backingfile.qcow2" echo " -h, --help Usage message" } error_invalid_params() { echo -e "${RED}[ERROR]Invalid params. Check the required params.${NO_COLOR}" usage exit 1 } while [[ "$#" -gt 0 ]]; do key="$1" case $key in --aws-access-key) aws_access_key="$2" shift # past argument shift # past value ;; --aws-secret-access-key) aws_secret_access_key="$2" shift # past argument shift # past value ;; --cifs-username) cifs_username="$2" shift # past argument shift # past value ;; --cifs-password) cifs_password="$2" shift # past argument shift # past value ;; -u|--backup-url) backup_url="$2" shift # past argument shift # past value ;; -o|--output-file) output_file="$2" shift # past argument shift # past value ;; -f|--output-format) output_format="$2" shift # past argument shift # past value ;; -b|--backing-file) backing_file="$2" shift # past argument shift # past value ;; -v|--version) version="$2" shift # past argument shift # past value ;; -h|--help) usage exit 0 shift ;; *) error_invalid_params ;; esac done # Check the required parameters exits if [ -z "${backup_url}" ] || [ -z "${output_file}" ] || [ -z "${output_format}" ] || [ -z "${version}" ]; then error_invalid_params fi if [[ "${backup_url}" =~ ^[Ss]3 ]]; then if [ -z "${aws_access_key}" ] || [ -z "${aws_secret_access_key}" ]; then error_invalid_params fi fi # Compose the docker arguments if [[ "${backup_url}" =~ ^[Ss]3 ]]; then CUSTOMIZED_ARGS="-e AWS_ACCESS_KEY_ID="${aws_access_key}" -e AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}" " else CUSTOMIZED_ARGS="--cap-add SYS_ADMIN --security-opt apparmor:unconfined --cap-add DAC_READ_SEARCH" fi if [[ "${backup_url}" =~ ^cifs ]]; then CUSTOMIZED_ARGS+=" -e CIFS_USERNAME=${cifs_username} -e CIFS_PASSWORD=${cifs_password} " fi # Start restoring a backup to an image file. docker run ${CUSTOMIZED_ARGS} -v /tmp/restore:/tmp/restore \ longhornio/longhorn-engine:"${version}" longhorn backup \ restore-to-file ""${backup_url}"" \ --output-file "/tmp/restore/${output_file}" \ --output-format "${output_format}" \ --backing-file "${backing_file}" ================================================ FILE: scripts/save-images.sh ================================================ #!/usr/bin/env bash list="longhorn-images.txt" CONTAINER_CLI=${CONTAINER_CLI:-docker} POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in -i|--images) images="$2" shift # past argument shift # past value ;; -l|--image-list) list="$2" shift # past argument shift # past value ;; -p|--platform) platform="$2" shift # past argument shift # past value ;; -h|--help) help="true" shift ;; *) echo "Error! invalid flag: ${key}" help="true" break ;; esac done usage () { echo "USAGE: $0 [--image-list longhorn-images.txt] [--images longhorn-images.tar.gz] [--platform linux/amd64]" echo " [-l|--images-list path] text file with list of images. 1 per line." echo " [-p|--platform linux/arch] if using images-list path, pulls the image with the specified platform" echo " [-i|--images path] tar.gz generated by docker/podman save. If this flag is empty, the script does not export images to a tar.gz file" echo " [-h|--help] Usage message" echo "" echo "To use podman instead of docker set the environment variable CONTAINER_CLI=podman" } if [[ $help ]]; then usage exit 0 fi set -e -x for i in $(cat ${list}); do if [ -n "$platform" ]; then $CONTAINER_CLI pull ${i} --platform $platform else $CONTAINER_CLI pull ${i} fi done if [[ $images ]]; then $CONTAINER_CLI save $(cat ${list} | tr '\n' ' ') | gzip -c > ${images} fi ================================================ FILE: scripts/update-chart-questions.sh ================================================ #!/usr/bin/env bash function die() { echo >&2 "$@" exit 1 } function check_yq() { if ! command -v yq >/dev/null; then die "Missing required program 'yq'." fi yq --version | grep mikefarah >/dev/null if [ $? -ne 0 ]; then die "yq exists but it's not the one we need (mikefarah/yq)." fi } check_yq while IFS= read -r line; do image="$line" repo="${image%:*}" tag="${image##*:}" component="${repo#longhornio/}" if [ "$component" = "csi-attacher" ]; then key="image.csi.attacher.tag" elif [ "$component" = "csi-provisioner" ]; then key="image.csi.provisioner.tag" elif [ "$component" = "csi-resizer" ]; then key="image.csi.resizer.tag" elif [ "$component" = "csi-snapshotter" ]; then key="image.csi.snapshotter.tag" elif [ "$component" = "csi-node-driver-registrar" ]; then key="image.csi.nodeDriverRegistrar.tag" elif [ "$component" = "livenessprobe" ]; then key="image.csi.livenessProbe.tag" elif [ "$component" = "backing-image-manager" ]; then key="image.longhorn.backingImageManager.tag" elif [ "$component" = "longhorn-engine" ]; then key="image.longhorn.engine.tag" elif [ "$component" = "longhorn-instance-manager" ]; then key="image.longhorn.instanceManager.tag" elif [ "$component" = "longhorn-manager" ]; then key="image.longhorn.manager.tag" elif [ "$component" = "longhorn-share-manager" ]; then key="image.longhorn.shareManager.tag" elif [ "$component" = "longhorn-ui" ]; then key="image.longhorn.ui.tag" elif [ "$component" = "longhorn-cli" ]; then key="image.longhorn.cli.tag" elif [ "$component" = "support-bundle-kit" ]; then key="image.longhorn.supportBundleKit.tag" else echo "Component $component is not found in the chart" continue fi yq eval -i ".questions[].subquestions[] |= (select(.variable == \"$key\") | .default = \"$tag\")" chart/questions.yaml done < "deploy/longhorn-images.txt" ================================================ FILE: scripts/update-chart-readme.sh ================================================ #!/usr/bin/env bash while IFS= read -r line; do image="$line" repo="${image%:*}" tag="${image##*:}" component="${repo#longhornio/}" if [ "$component" = "csi-attacher" ]; then key="image.csi.attacher.tag" elif [ "$component" = "csi-provisioner" ]; then key="image.csi.provisioner.tag" elif [ "$component" = "csi-resizer" ]; then key="image.csi.resizer.tag" elif [ "$component" = "csi-snapshotter" ]; then key="image.csi.snapshotter.tag" elif [ "$component" = "csi-node-driver-registrar" ]; then key="image.csi.nodeDriverRegistrar.tag" elif [ "$component" = "livenessprobe" ]; then key="image.csi.livenessProbe.tag" elif [ "$component" = "backing-image-manager" ]; then key="image.longhorn.backingImageManager.tag" elif [ "$component" = "longhorn-engine" ]; then key="image.longhorn.engine.tag" elif [ "$component" = "longhorn-instance-manager" ]; then key="image.longhorn.instanceManager.tag" elif [ "$component" = "longhorn-manager" ]; then key="image.longhorn.manager.tag" elif [ "$component" = "longhorn-share-manager" ]; then key="image.longhorn.shareManager.tag" elif [ "$component" = "longhorn-ui" ]; then key="image.longhorn.ui.tag" elif [ "$component" = "longhorn-cli" ]; then key="image.longhorn.cli.tag" elif [ "$component" = "support-bundle-kit" ]; then key="image.longhorn.supportBundleKit.tag" else echo "Component $component is not found in the chart" continue fi new_default="\"$tag\"" escaped_new_default=$(printf '%s' \`"$new_default"\` | sed -e 's/[\/&]/\\&/g') escaped_key=$(printf '%s' "$key" | sed 's/\./\\./g') # Update the default value in the chart's README.md and save to a temporary file sed "s/\(|[ ]*$escaped_key[ ]*|[ ]*string[ ]*|[ ]*\)[^|]*\(|.*\)/\1$escaped_new_default \2/" chart/README.md > chart/README.md.tmp mv chart/README.md.tmp chart/README.md done < "deploy/longhorn-images.txt" ================================================ FILE: scripts/update-chart-values.sh ================================================ #!/usr/bin/env bash function die() { echo >&2 "$@" exit 1 } function check_yq() { if ! command -v yq >/dev/null; then die "Missing required program 'yq'." fi yq --version | grep mikefarah >/dev/null if [ $? -ne 0 ]; then die "yq exists but it's not the one we need (mikefarah/yq)." fi } check_yq while IFS= read -r line; do image="$line" repo="${image%:*}" tag="${image##*:}" component="${repo#longhornio/}" if [ "$component" = "csi-attacher" ]; then key=".image.csi.attacher.tag" elif [ "$component" = "csi-provisioner" ]; then key=".image.csi.provisioner.tag" elif [ "$component" = "csi-resizer" ]; then key=".image.csi.resizer.tag" elif [ "$component" = "csi-snapshotter" ]; then key=".image.csi.snapshotter.tag" elif [ "$component" = "csi-node-driver-registrar" ]; then key=".image.csi.nodeDriverRegistrar.tag" elif [ "$component" = "livenessprobe" ]; then key=".image.csi.livenessProbe.tag" elif [ "$component" = "support-bundle-kit" ]; then key=".image.longhorn.supportBundleKit.tag" else echo "Component $component is not found in the chart" continue fi yq -i "$key = \"$tag\"" chart/values.yaml done < "deploy/longhorn-images.txt" ================================================ FILE: scripts/update-manifests-dev-version.sh ================================================ #!/usr/bin/env bash # Example: # ./scripts/update-manifests-dev-version.sh 1.3.0 1.4.0 # # Result: # - Chart version will be updated to 1.4.0-dev # - Images (manager, engine, ui) will be updated to master-head set -o errexit set -o nounset PRJ_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null || realpath "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null) CURRENT_VERSION=${CURRENT_VERSION:-$1} NEW_VERSION=${NEW_VERSION:-$2}-dev mapfile -t manifests < <(find "$PRJ_DIR" -type f -a \( -name '*.yaml' -o -name 'longhorn-images.txt' \)) if [[ ${#manifests[@]} -le 0 ]]; then echo "No manifests found to update from $PRJ_DIR" >/dev/stderr exit 1 fi echo "Updating $CURRENT_VERSION -> $NEW_VERSION-dev with master-head images in below manifests" for f in "${manifests[@]}"; do f_name=$(basename "$f") if [[ $f_name == "Chart.yaml" ]]; then sed -i "s#\(version: \)${CURRENT_VERSION}#\1${NEW_VERSION}#g" "$f" sed -i "s#\(appVersion: v\)${CURRENT_VERSION}#\1${NEW_VERSION}#g" "$f" else sed -i "s#\(:\s*\)v${CURRENT_VERSION}#\1master-head#g" "$f" fi echo "$f updated" done . "$PRJ_DIR"/scripts/generate-longhorn-yaml.sh ================================================ FILE: scripts/update-uninstall-manifest.py ================================================ import yaml import os from typing import List, Dict, Iterator from pathlib import Path class YAMLResourceProcessor: def __init__(self, longhorn_manifest: str, uninstall_file: str): self.longhorn_manifest = Path(longhorn_manifest) self.uninstall_file = Path(uninstall_file) self.output_file = self.uninstall_file.parent / 'uninstall.tmp.yaml' def load_yaml_documents(self, file_path: Path) -> Iterator[Dict]: """Load YAML documents from a file, skipping None values.""" try: with open(file_path, 'r') as file: for doc in yaml.safe_load_all(file): if doc is not None: yield doc except (yaml.YAMLError, OSError) as e: raise RuntimeError(f"Error processing {file_path}: {str(e)}") def extract_crd_resources(self) -> List[str]: """Extract CustomResourceDefinition names from the manifest.""" resources = [] for doc in self.load_yaml_documents(self.longhorn_manifest): if doc.get('kind') == "CustomResourceDefinition": name = doc.get('metadata', {}).get('name', '') if name: # Remove '.longhorn.io' suffix resources.append(name.split(".")[0]) return resources def update_cluster_role(self, doc: Dict, resources: List[str]) -> Dict: """Update ClusterRole document with new resources.""" if doc['kind'] == "ClusterRole": for rule in doc.get('rules', []): if rule.get('apiGroups') == ["longhorn.io"]: rule['resources'] = resources return doc def process_files(self) -> None: """Process the input files and generate the output file.""" resources = self.extract_crd_resources() try: with open(self.output_file, 'w') as output: for doc in self.load_yaml_documents(self.uninstall_file): updated_doc = self.update_cluster_role(doc, resources) yaml.dump(updated_doc, output) output.write('---\n') except (yaml.YAMLError, OSError) as e: raise RuntimeError(f"Error writing output file: {str(e)}") def main(): try: processor = YAMLResourceProcessor( longhorn_manifest='deploy/longhorn.yaml', uninstall_file='uninstall/uninstall.yaml' ) processor.process_files() # Replace the original file with the updated file os.replace(processor.output_file, processor.uninstall_file) print("YAML processing completed successfully.") except Exception as e: print(f"Error: {str(e)}") exit(1) if __name__ == "__main__": main() ================================================ FILE: support-versions.txt ================================================ v1.10.2 v1.9.2 v1.8.2 ================================================ FILE: uninstall/uninstall.yaml ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-uninstall-service-account namespace: longhorn-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: longhorn-uninstall-role rules: - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - '*' - apiGroups: - '' resources: - pods - persistentvolumes - persistentvolumeclaims - nodes - configmaps - secrets - services - endpoints verbs: - '*' - apiGroups: - apps resources: - daemonsets - statefulsets - deployments verbs: - '*' - apiGroups: - batch resources: - jobs - cronjobs verbs: - '*' - apiGroups: - policy resources: - poddisruptionbudgets verbs: - '*' - apiGroups: - scheduling.k8s.io resources: - priorityclasses verbs: - watch - list - apiGroups: - storage.k8s.io resources: - csidrivers - storageclasses - volumeattachments verbs: - '*' - apiGroups: - longhorn.io resources: - backingimagedatasources - backingimagemanagers - backingimages - backupbackingimages - backups - backuptargets - backupvolumes - engineimages - engines - instancemanagers - nodes - orphans - recurringjobs - replicas - settings - sharemanagers - snapshots - supportbundles - systembackups - systemrestores - volumeattachments - volumes verbs: - '*' - apiGroups: - coordination.k8s.io resources: - leases verbs: - '*' - apiGroups: - admissionregistration.k8s.io resources: - mutatingwebhookconfigurations - validatingwebhookconfigurations verbs: - get - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: longhorn-uninstall-bind roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: longhorn-uninstall-role subjects: - kind: ServiceAccount name: longhorn-uninstall-service-account namespace: longhorn-system --- apiVersion: batch/v1 kind: Job metadata: name: longhorn-uninstall namespace: longhorn-system spec: activeDeadlineSeconds: 900 backoffLimit: 1 template: metadata: name: longhorn-uninstall spec: containers: - command: - longhorn-manager - uninstall - --force env: - name: LONGHORN_NAMESPACE value: longhorn-system image: longhornio/longhorn-manager:master-head imagePullPolicy: IfNotPresent name: longhorn-uninstall restartPolicy: Never serviceAccountName: longhorn-uninstall-service-account ---